You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by de...@apache.org on 2026/05/26 13:37:08 UTC
(beam) branch 20260524_huggingFaceTry#3 created (now b1676adfd46)
This is an automated email from the ASF dual-hosted git repository.
derrickaw pushed a change to branch 20260524_huggingFaceTry#3
in repository https://gitbox.apache.org/repos/asf/beam.git
at b1676adfd46 Fix installYamlIntegrationTestDeps to use constraint files and prevent pip backtracking hangs
This branch includes the following new commits:
new 57d08ed5365 huggingface model handler for yaml
new 4559bf9871f add yaml huggingface test file
new c5181114e08 update dependency logic
new b1676adfd46 Fix installYamlIntegrationTestDeps to use constraint files and prevent pip backtracking hangs
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(beam) 03/04: update dependency logic
Posted by de...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
derrickaw pushed a commit to branch 20260524_huggingFaceTry#3
in repository https://gitbox.apache.org/repos/asf/beam.git
commit c5181114e0825ee908aeb82499187af308f42979
Author: Derrick Williams <de...@google.com>
AuthorDate: Tue May 12 02:45:51 2026 +0000
update dependency logic
---
sdks/python/build.gradle | 17 ++++-------------
1 file changed, 4 insertions(+), 13 deletions(-)
diff --git a/sdks/python/build.gradle b/sdks/python/build.gradle
index 8450e32d593..837631868b8 100644
--- a/sdks/python/build.gradle
+++ b/sdks/python/build.gradle
@@ -131,19 +131,10 @@ tasks.register("installYamlIntegrationTestDeps") {
executable 'sh'
args '-c', ". ${envdir}/bin/activate && " +
"py_ver=\$(python -c 'import sys; print(f\"{sys.version_info.major}{sys.version_info.minor}\")') && " +
- "constraint_file=\"\" && " +
- "if [ -f \"container/ml/py\${py_ver}/base_image_requirements.txt\" ]; then " +
- " constraint_file=\"container/ml/py\${py_ver}/base_image_requirements.txt\"; " +
- "elif [ -f \"container/py\${py_ver}/base_image_requirements.txt\" ]; then " +
- " constraint_file=\"container/py\${py_ver}/base_image_requirements.txt\"; " +
- "fi && " +
- "if [ -n \"\$constraint_file\" ]; then " +
- " echo \"Installing with constraint file: \$constraint_file\" && " +
- " pip install --pre --retries 10 -c \"\$constraint_file\" \${buildDir}/apache-beam.tar.gz[ml_test,yaml,transformers]; " +
- "else " +
- " echo \"No constraint file found, installing without constraints\" && " +
- " pip install --pre --retries 10 \${buildDir}/apache-beam.tar.gz[ml_test,yaml,transformers]; " +
- "fi"
+ "ml_extra=\"ml_test\" && " +
+ "if [ \"\$py_ver\" -ge 313 ]; then ml_extra=\"p\${py_ver}_ml_test\"; fi && " +
+ "echo \"Installing dependencies...\" && " +
+ "pip install --pre --retries 10 ${buildDir}/apache-beam.tar.gz[\$ml_extra,yaml,transformers]"
}
}
}
(beam) 04/04: Fix installYamlIntegrationTestDeps to use constraint files and prevent pip backtracking hangs
Posted by de...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
derrickaw pushed a commit to branch 20260524_huggingFaceTry#3
in repository https://gitbox.apache.org/repos/asf/beam.git
commit b1676adfd4648f6e5794ec9c04b26abb3bc56048
Author: Derrick Williams <de...@google.com>
AuthorDate: Sun May 24 18:20:12 2026 +0000
Fix installYamlIntegrationTestDeps to use constraint files and prevent pip backtracking hangs
---
sdks/python/build.gradle | 15 +++++++++++++--
1 file changed, 13 insertions(+), 2 deletions(-)
diff --git a/sdks/python/build.gradle b/sdks/python/build.gradle
index 837631868b8..0ae4aaa4148 100644
--- a/sdks/python/build.gradle
+++ b/sdks/python/build.gradle
@@ -133,8 +133,19 @@ tasks.register("installYamlIntegrationTestDeps") {
"py_ver=\$(python -c 'import sys; print(f\"{sys.version_info.major}{sys.version_info.minor}\")') && " +
"ml_extra=\"ml_test\" && " +
"if [ \"\$py_ver\" -ge 313 ]; then ml_extra=\"p\${py_ver}_ml_test\"; fi && " +
- "echo \"Installing dependencies...\" && " +
- "pip install --pre --retries 10 ${buildDir}/apache-beam.tar.gz[\$ml_extra,yaml,transformers]"
+ "constraint_file=\"\" && " +
+ "if [ -f \"container/ml/py\${py_ver}/base_image_requirements.txt\" ]; then " +
+ " constraint_file=\"container/ml/py\${py_ver}/base_image_requirements.txt\"; " +
+ "elif [ -f \"container/py\${py_ver}/base_image_requirements.txt\" ]; then " +
+ " constraint_file=\"container/py\${py_ver}/base_image_requirements.txt\"; " +
+ "fi && " +
+ "if [ -n \"\$constraint_file\" ]; then " +
+ " echo \"Installing with constraint file: \$constraint_file\" && " +
+ " pip install --pre --retries 10 -c \"\$constraint_file\" \${buildDir}/apache-beam.tar.gz[\$ml_extra,yaml,transformers]; " +
+ "else " +
+ " echo \"No constraint file found, installing without constraints\" && " +
+ " pip install --pre --retries 10 \${buildDir}/apache-beam.tar.gz[\$ml_extra,yaml,transformers]; " +
+ "fi"
}
}
}
(beam) 02/04: add yaml huggingface test file
Posted by de...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
derrickaw pushed a commit to branch 20260524_huggingFaceTry#3
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 4559bf9871f1f49b675381f3cef45b545027e4d3
Author: Derrick Williams <de...@google.com>
AuthorDate: Mon May 11 19:34:59 2026 +0000
add yaml huggingface test file
---
.../yaml/tests/runinference_huggingface.yaml | 62 ++++++++++++++++++++++
1 file changed, 62 insertions(+)
diff --git a/sdks/python/apache_beam/yaml/tests/runinference_huggingface.yaml b/sdks/python/apache_beam/yaml/tests/runinference_huggingface.yaml
new file mode 100644
index 00000000000..8728a6f544a
--- /dev/null
+++ b/sdks/python/apache_beam/yaml/tests/runinference_huggingface.yaml
@@ -0,0 +1,62 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+pipelines:
+ - pipeline:
+ type: chain
+ transforms:
+ - type: Create
+ config:
+ elements:
+ - text: "I love Apache Beam!"
+ - text: "I hate this error."
+ - type: RunInference
+ config:
+ model_handler:
+ type: "HuggingFacePipeline"
+ config:
+ task: "text-classification"
+ inference_fn:
+ callable: |
+ def real_inference(batch, pipeline, inference_args):
+ predictions = pipeline(batch, **inference_args)
+
+ # If it's a single dictionary (batch size of 1), wrap it in a list
+ if isinstance(predictions, dict):
+ predictions = [predictions]
+
+ return {
+ 'label': [p['label'] for p in predictions],
+ 'score': [p['score'] for p in predictions]
+ }
+ preprocess:
+ callable: 'lambda x: x.text'
+ - type: MapToFields
+ config:
+ language: python
+ fields:
+ text: text
+ sentiment:
+ callable: 'lambda x: x.inference.inference["label"]'
+ - type: AssertEqual
+ config:
+ elements:
+ - text: "I love Apache Beam!"
+ sentiment: "POSITIVE"
+ - text: "I hate this error."
+ sentiment: "NEGATIVE"
+
+ options:
+ yaml_experimental_features: ['ML']
(beam) 01/04: huggingface model handler for yaml
Posted by de...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
derrickaw pushed a commit to branch 20260524_huggingFaceTry#3
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 57d08ed5365934c03c80b6554e2e489c2bf5385f
Author: Derrick Williams <de...@google.com>
AuthorDate: Mon May 11 19:21:44 2026 +0000
huggingface model handler for yaml
---
.../beam_PostCommit_Yaml_Xlang_Direct.json | 2 +-
.../beam_PostCommit_Yaml_Xlang_Direct.yml | 2 +-
.../workflows/beam_PreCommit_Yaml_Xlang_Direct.yml | 2 +-
...uninference.yaml => runinference_vertexai.yaml} | 0
sdks/python/apache_beam/yaml/yaml_ml.py | 49 ++++++++++++++++++++++
sdks/python/build.gradle | 28 ++++++++++++-
6 files changed, 78 insertions(+), 5 deletions(-)
diff --git a/.github/trigger_files/beam_PostCommit_Yaml_Xlang_Direct.json b/.github/trigger_files/beam_PostCommit_Yaml_Xlang_Direct.json
index 541dc4ea8e8..8ed972c9f57 100644
--- a/.github/trigger_files/beam_PostCommit_Yaml_Xlang_Direct.json
+++ b/.github/trigger_files/beam_PostCommit_Yaml_Xlang_Direct.json
@@ -1,4 +1,4 @@
{
"comment": "Modify this file in a trivial way to cause this test suite to run",
- "revision": 2
+ "revision": 3
}
diff --git a/.github/workflows/beam_PostCommit_Yaml_Xlang_Direct.yml b/.github/workflows/beam_PostCommit_Yaml_Xlang_Direct.yml
index ea1c255f7cc..afa437b64de 100644
--- a/.github/workflows/beam_PostCommit_Yaml_Xlang_Direct.yml
+++ b/.github/workflows/beam_PostCommit_Yaml_Xlang_Direct.yml
@@ -80,7 +80,7 @@ jobs:
- name: run PostCommit Yaml Xlang Direct script
uses: ./.github/actions/gradle-command-self-hosted-action
with:
- gradle-command: :sdks:python:postCommitYamlIntegrationTests -PyamlTestSet=${{ matrix.test_set }} -PbeamPythonExtra=ml_test,yaml
+ gradle-command: :sdks:python:postCommitYamlIntegrationTests -PyamlTestSet=${{ matrix.test_set }}
- name: Archive Python Test Results
uses: actions/upload-artifact@v7
if: failure()
diff --git a/.github/workflows/beam_PreCommit_Yaml_Xlang_Direct.yml b/.github/workflows/beam_PreCommit_Yaml_Xlang_Direct.yml
index 34a1af741f7..bde75568101 100644
--- a/.github/workflows/beam_PreCommit_Yaml_Xlang_Direct.yml
+++ b/.github/workflows/beam_PreCommit_Yaml_Xlang_Direct.yml
@@ -91,7 +91,7 @@ jobs:
- name: run PreCommit Yaml Xlang Direct script
uses: ./.github/actions/gradle-command-self-hosted-action
with:
- gradle-command: :sdks:python:yamlIntegrationTests -PbeamPythonExtra=ml_test,yaml
+ gradle-command: :sdks:python:yamlIntegrationTests
- name: Archive Python Test Results
uses: actions/upload-artifact@v7
if: failure()
diff --git a/sdks/python/apache_beam/yaml/tests/runinference.yaml b/sdks/python/apache_beam/yaml/tests/runinference_vertexai.yaml
similarity index 100%
rename from sdks/python/apache_beam/yaml/tests/runinference.yaml
rename to sdks/python/apache_beam/yaml/tests/runinference_vertexai.yaml
diff --git a/sdks/python/apache_beam/yaml/yaml_ml.py b/sdks/python/apache_beam/yaml/yaml_ml.py
index 51f18c73304..05cbed3bd45 100644
--- a/sdks/python/apache_beam/yaml/yaml_ml.py
+++ b/sdks/python/apache_beam/yaml/yaml_ml.py
@@ -282,6 +282,55 @@ class VertexAIModelHandlerJSONProvider(ModelHandlerProvider):
('model_id', Optional[str])])
+@ModelHandlerProvider.register_handler_type('HuggingFacePipeline')
+class HuggingFacePipelineProvider(ModelHandlerProvider):
+ def __init__(
+ self,
+ task: Optional[str] = None,
+ model: Optional[str] = None,
+ preprocess: Optional[dict[str, str]] = None,
+ postprocess: Optional[dict[str, str]] = None,
+ device: Optional[Any] = None,
+ inference_fn: Optional[dict[str, str]] = None,
+ load_pipeline_args: Optional[dict[str, Any]] = None,
+ **kwargs):
+ try:
+ from apache_beam.ml.inference.huggingface_inference import HuggingFacePipelineModelHandler
+ except ImportError:
+ raise ValueError(
+ 'Unable to import HuggingFacePipelineModelHandler. Please '
+ 'install transformers dependencies.')
+
+ kwargs = {k: v for k, v in kwargs.items() if not k.startswith('_')}
+
+ inference_fn_obj = self.parse_processing_transform(
+ inference_fn, 'inference_fn') if inference_fn else None
+
+ handler_kwargs = {}
+ if inference_fn_obj:
+ handler_kwargs['inference_fn'] = inference_fn_obj
+
+ _handler = HuggingFacePipelineModelHandler(
+ task=task,
+ model=model,
+ device=device,
+ load_pipeline_args=load_pipeline_args,
+ **handler_kwargs,
+ **kwargs)
+
+ super().__init__(_handler, preprocess, postprocess)
+
+ @staticmethod
+ def validate(config):
+ if not config.get('task') and not config.get('model'):
+ raise ValueError(
+ "HuggingFacePipeline requires either 'task' or "
+ "'model' to be specified.")
+
+ def inference_output_type(self):
+ return Any
+
+
@beam.ptransform.ptransform_fn
def run_inference(
pcoll,
diff --git a/sdks/python/build.gradle b/sdks/python/build.gradle
index 5f09dff57e8..8450e32d593 100644
--- a/sdks/python/build.gradle
+++ b/sdks/python/build.gradle
@@ -124,10 +124,34 @@ tasks.register("generateYamlDocs") {
outputs.file "${buildDir}/yaml-examples.html"
}
+tasks.register("installYamlIntegrationTestDeps") {
+ dependsOn installGcpTest
+ doLast {
+ exec {
+ executable 'sh'
+ args '-c', ". ${envdir}/bin/activate && " +
+ "py_ver=\$(python -c 'import sys; print(f\"{sys.version_info.major}{sys.version_info.minor}\")') && " +
+ "constraint_file=\"\" && " +
+ "if [ -f \"container/ml/py\${py_ver}/base_image_requirements.txt\" ]; then " +
+ " constraint_file=\"container/ml/py\${py_ver}/base_image_requirements.txt\"; " +
+ "elif [ -f \"container/py\${py_ver}/base_image_requirements.txt\" ]; then " +
+ " constraint_file=\"container/py\${py_ver}/base_image_requirements.txt\"; " +
+ "fi && " +
+ "if [ -n \"\$constraint_file\" ]; then " +
+ " echo \"Installing with constraint file: \$constraint_file\" && " +
+ " pip install --pre --retries 10 -c \"\$constraint_file\" \${buildDir}/apache-beam.tar.gz[ml_test,yaml,transformers]; " +
+ "else " +
+ " echo \"No constraint file found, installing without constraints\" && " +
+ " pip install --pre --retries 10 \${buildDir}/apache-beam.tar.gz[ml_test,yaml,transformers]; " +
+ "fi"
+ }
+ }
+}
+
tasks.register("yamlIntegrationTests") {
description "Runs precommit integration tests for yaml pipelines."
- dependsOn installGcpTest
+ dependsOn installYamlIntegrationTestDeps
// Need to build all expansion services referenced in apache_beam/yaml/*.*
// grep -oh 'sdk.*Jar' sdks/python/apache_beam/yaml/*.yaml | sort | uniq
dependsOn ":sdks:java:extensions:schemaio-expansion-service:shadowJar"
@@ -146,7 +170,7 @@ tasks.register("yamlIntegrationTests") {
tasks.register("postCommitYamlIntegrationTests") {
description "Runs postcommit integration tests for yaml pipelines - parameterized by yamlTestSet."
- dependsOn installGcpTest
+ dependsOn installYamlIntegrationTestDeps
// Need to build all expansion services referenced in apache_beam/yaml/*.*
// grep -oh 'sdk.*Jar' sdks/python/apache_beam/yaml/*.yaml | sort | uniq
dependsOn ":sdks:java:extensions:schemaio-expansion-service:shadowJar"