You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ri...@apache.org on 2024/03/29 00:16:23 UTC
(beam) branch master updated: [Python] Add redis client to python dependencies (#30677)
This is an automated email from the ASF dual-hosted git repository.
riteshghorse pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 25805db9278 [Python] Add redis client to python dependencies (#30677)
25805db9278 is described below
commit 25805db92786478d959ee3e2a1943e584e75c304
Author: Ritesh Ghorse <ri...@gmail.com>
AuthorDate: Thu Mar 28 20:16:16 2024 -0400
[Python] Add redis client to python dependencies (#30677)
* add redis client to python dependencies
* fix lint
* update dependencies
* add upper bound
* remove redis requirements file and uses_redis tag
* new task for testcontainers
* fix expected value after syncing
* remove requirement file
---
.github/trigger_files/beam_PostCommit_Python.json | 0
.../snippets/transforms/elementwise/enrichment_test.py | 6 +++---
sdks/python/apache_beam/io/requestresponse.py | 2 +-
sdks/python/apache_beam/io/requestresponse_it_test.py | 2 +-
.../io/requestresponse_tests_requirements.txt | 18 ------------------
.../transforms/enrichment_handlers/bigtable_it_test.py | 2 +-
.../vertex_ai_feature_store_it_test.py | 2 +-
.../vertex_ai_feature_store_tests_requirement.txt | 18 ------------------
.../python/container/py310/base_image_requirements.txt | 4 +++-
.../python/container/py311/base_image_requirements.txt | 3 ++-
sdks/python/container/py38/base_image_requirements.txt | 4 +++-
sdks/python/container/py39/base_image_requirements.txt | 4 +++-
sdks/python/pytest.ini | 2 +-
sdks/python/setup.py | 1 +
sdks/python/test-suites/dataflow/common.gradle | 9 +--------
sdks/python/test-suites/direct/common.gradle | 16 +++++-----------
16 files changed, 26 insertions(+), 67 deletions(-)
diff --git a/.github/trigger_files/beam_PostCommit_Python.json b/.github/trigger_files/beam_PostCommit_Python.json
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/sdks/python/apache_beam/examples/snippets/transforms/elementwise/enrichment_test.py b/sdks/python/apache_beam/examples/snippets/transforms/elementwise/enrichment_test.py
index 3fd759bbc32..8a7cdfbe926 100644
--- a/sdks/python/apache_beam/examples/snippets/transforms/elementwise/enrichment_test.py
+++ b/sdks/python/apache_beam/examples/snippets/transforms/elementwise/enrichment_test.py
@@ -44,9 +44,9 @@ Row(sale_id=5, customer_id=5, product_id=4, quantity=2, product={'product_id': '
def validate_enrichment_with_vertex_ai():
expected = '''[START enrichment_with_vertex_ai]
-Row(user_id='2963', product_id=14235, sale_price=15.0, age=29.0, gender='1', state='97', country='2')
-Row(user_id='21422', product_id=11203, sale_price=12.0, age=36.0, state='184', gender='1', country='5')
-Row(user_id='20592', product_id=8579, sale_price=9.0, age=30.0, state='86', gender='1', country='4')
+Row(user_id='2963', product_id=14235, sale_price=15.0, age=12.0, state='1', gender='1', country='1')
+Row(user_id='21422', product_id=11203, sale_price=12.0, age=12.0, state='0', gender='0', country='0')
+Row(user_id='20592', product_id=8579, sale_price=9.0, age=12.0, state='2', gender='1', country='2')
[END enrichment_with_vertex_ai]'''.splitlines()[1:-1]
return expected
diff --git a/sdks/python/apache_beam/io/requestresponse.py b/sdks/python/apache_beam/io/requestresponse.py
index 706bce95f5e..4458aa59c18 100644
--- a/sdks/python/apache_beam/io/requestresponse.py
+++ b/sdks/python/apache_beam/io/requestresponse.py
@@ -33,10 +33,10 @@ from typing import Tuple
from typing import TypeVar
from typing import Union
+import redis
from google.api_core.exceptions import TooManyRequests
import apache_beam as beam
-import redis
from apache_beam import pvalue
from apache_beam.coders import coders
from apache_beam.io.components.adaptive_throttler import AdaptiveThrottler
diff --git a/sdks/python/apache_beam/io/requestresponse_it_test.py b/sdks/python/apache_beam/io/requestresponse_it_test.py
index bd8c63dea58..712ccc7881d 100644
--- a/sdks/python/apache_beam/io/requestresponse_it_test.py
+++ b/sdks/python/apache_beam/io/requestresponse_it_test.py
@@ -203,7 +203,7 @@ class FakeCallerForCache(Caller[str, str]):
pass
-@pytest.mark.uses_redis
+@pytest.mark.uses_testcontainer
class TestRedisCache(unittest.TestCase):
def setUp(self) -> None:
self.retries = 3
diff --git a/sdks/python/apache_beam/io/requestresponse_tests_requirements.txt b/sdks/python/apache_beam/io/requestresponse_tests_requirements.txt
deleted file mode 100644
index 1d886970509..00000000000
--- a/sdks/python/apache_beam/io/requestresponse_tests_requirements.txt
+++ /dev/null
@@ -1,18 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-redis>=5.0.0
diff --git a/sdks/python/apache_beam/transforms/enrichment_handlers/bigtable_it_test.py b/sdks/python/apache_beam/transforms/enrichment_handlers/bigtable_it_test.py
index b792bc8ba94..bbabbc306e6 100644
--- a/sdks/python/apache_beam/transforms/enrichment_handlers/bigtable_it_test.py
+++ b/sdks/python/apache_beam/transforms/enrichment_handlers/bigtable_it_test.py
@@ -148,7 +148,7 @@ def create_rows(table):
row.commit()
-@pytest.mark.uses_redis
+@pytest.mark.uses_testcontainer
class TestBigTableEnrichment(unittest.TestCase):
def setUp(self):
self.project_id = 'apache-beam-testing'
diff --git a/sdks/python/apache_beam/transforms/enrichment_handlers/vertex_ai_feature_store_it_test.py b/sdks/python/apache_beam/transforms/enrichment_handlers/vertex_ai_feature_store_it_test.py
index 066553decf2..c5482309a25 100644
--- a/sdks/python/apache_beam/transforms/enrichment_handlers/vertex_ai_feature_store_it_test.py
+++ b/sdks/python/apache_beam/transforms/enrichment_handlers/vertex_ai_feature_store_it_test.py
@@ -62,7 +62,7 @@ class ValidateResponse(beam.DoFn):
f"from feature store")
-@pytest.mark.uses_redis
+@pytest.mark.uses_testcontainer
class TestVertexAIFeatureStoreHandler(unittest.TestCase):
def setUp(self) -> None:
self.project = 'apache-beam-testing'
diff --git a/sdks/python/apache_beam/transforms/enrichment_handlers/vertex_ai_feature_store_tests_requirement.txt b/sdks/python/apache_beam/transforms/enrichment_handlers/vertex_ai_feature_store_tests_requirement.txt
deleted file mode 100644
index cd74683a51c..00000000000
--- a/sdks/python/apache_beam/transforms/enrichment_handlers/vertex_ai_feature_store_tests_requirement.txt
+++ /dev/null
@@ -1,18 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-redis
-google-cloud-aiplatform
diff --git a/sdks/python/container/py310/base_image_requirements.txt b/sdks/python/container/py310/base_image_requirements.txt
index 1258575696d..06fd375a831 100644
--- a/sdks/python/container/py310/base_image_requirements.txt
+++ b/sdks/python/container/py310/base_image_requirements.txt
@@ -21,6 +21,7 @@
# https://s.apache.org/beam-python-dev-wiki
# Reach out to a committer if you need help.
+async-timeout==4.0.3
attrs==23.2.0
beautifulsoup4==4.12.3
bs4==0.0.2
@@ -81,7 +82,7 @@ grpcio-status==1.62.1
guppy3==3.1.4.post1
hdfs==2.7.3
httplib2==0.22.0
-hypothesis==6.99.12
+hypothesis==6.99.13
idna==3.6
iniconfig==2.0.0
joblib==1.3.2
@@ -124,6 +125,7 @@ python-dateutil==2.9.0.post0
python-snappy==0.7.1
pytz==2024.1
PyYAML==6.0.1
+redis==5.0.3
referencing==0.34.0
regex==2023.12.25
requests==2.31.0
diff --git a/sdks/python/container/py311/base_image_requirements.txt b/sdks/python/container/py311/base_image_requirements.txt
index 5c703f727b9..71b3c48a35b 100644
--- a/sdks/python/container/py311/base_image_requirements.txt
+++ b/sdks/python/container/py311/base_image_requirements.txt
@@ -79,7 +79,7 @@ grpcio-status==1.62.1
guppy3==3.1.4.post1
hdfs==2.7.3
httplib2==0.22.0
-hypothesis==6.99.12
+hypothesis==6.99.13
idna==3.6
iniconfig==2.0.0
joblib==1.3.2
@@ -121,6 +121,7 @@ pytest-xdist==3.5.0
python-dateutil==2.9.0.post0
pytz==2024.1
PyYAML==6.0.1
+redis==5.0.3
referencing==0.34.0
regex==2023.12.25
requests==2.31.0
diff --git a/sdks/python/container/py38/base_image_requirements.txt b/sdks/python/container/py38/base_image_requirements.txt
index 9a6e3af7953..887aa8a2531 100644
--- a/sdks/python/container/py38/base_image_requirements.txt
+++ b/sdks/python/container/py38/base_image_requirements.txt
@@ -21,6 +21,7 @@
# https://s.apache.org/beam-python-dev-wiki
# Reach out to a committer if you need help.
+async-timeout==4.0.3
attrs==23.2.0
backports.zoneinfo==0.2.1
beautifulsoup4==4.12.3
@@ -82,7 +83,7 @@ grpcio-status==1.62.1
guppy3==3.1.4.post1
hdfs==2.7.3
httplib2==0.22.0
-hypothesis==6.99.12
+hypothesis==6.99.13
idna==3.6
importlib_metadata==7.1.0
importlib_resources==6.4.0
@@ -128,6 +129,7 @@ python-dateutil==2.9.0.post0
python-snappy==0.7.1
pytz==2024.1
PyYAML==6.0.1
+redis==5.0.3
referencing==0.34.0
regex==2023.12.25
requests==2.31.0
diff --git a/sdks/python/container/py39/base_image_requirements.txt b/sdks/python/container/py39/base_image_requirements.txt
index cf116324eac..9d8009c0048 100644
--- a/sdks/python/container/py39/base_image_requirements.txt
+++ b/sdks/python/container/py39/base_image_requirements.txt
@@ -21,6 +21,7 @@
# https://s.apache.org/beam-python-dev-wiki
# Reach out to a committer if you need help.
+async-timeout==4.0.3
attrs==23.2.0
beautifulsoup4==4.12.3
bs4==0.0.2
@@ -81,7 +82,7 @@ grpcio-status==1.62.1
guppy3==3.1.4.post1
hdfs==2.7.3
httplib2==0.22.0
-hypothesis==6.99.12
+hypothesis==6.99.13
idna==3.6
importlib_metadata==7.1.0
iniconfig==2.0.0
@@ -125,6 +126,7 @@ python-dateutil==2.9.0.post0
python-snappy==0.7.1
pytz==2024.1
PyYAML==6.0.1
+redis==5.0.3
referencing==0.34.0
regex==2023.12.25
requests==2.31.0
diff --git a/sdks/python/pytest.ini b/sdks/python/pytest.ini
index e78697169bb..8df74adbc62 100644
--- a/sdks/python/pytest.ini
+++ b/sdks/python/pytest.ini
@@ -65,7 +65,7 @@ markers =
uses_tf: tests that utilize tensorflow.
uses_transformers: tests that utilize transformers in some way.
vertex_ai_postcommit: vertex ai postcommits that need additional deps.
- uses_redis: enrichment transform tests that need redis.
+ uses_testcontainer: tests that use testcontainers.
uses_mock_api: tests that uses the mock API cluster.
# Default timeout intended for unit tests.
diff --git a/sdks/python/setup.py b/sdks/python/setup.py
index 409951cbc41..0e49f913fb4 100644
--- a/sdks/python/setup.py
+++ b/sdks/python/setup.py
@@ -392,6 +392,7 @@ if __name__ == '__main__':
'pydot>=1.2.0,<2',
'python-dateutil>=2.8.0,<3',
'pytz>=2018.3',
+ 'redis>=5.0.0,<6',
'regex>=2020.6.8',
'requests>=2.24.0,<3.0.0',
'typing-extensions>=3.7.0',
diff --git a/sdks/python/test-suites/dataflow/common.gradle b/sdks/python/test-suites/dataflow/common.gradle
index cadf3a6ae2c..8f3402035eb 100644
--- a/sdks/python/test-suites/dataflow/common.gradle
+++ b/sdks/python/test-suites/dataflow/common.gradle
@@ -491,13 +491,7 @@ task tftTests {
task mockAPITests {
dependsOn 'initializeForDataflowJob'
dependsOn ':sdks:python:sdist'
- def requirementsFile = "${rootDir}/sdks/python/apache_beam/io/requestresponse_tests_requirements.txt"
- doFirst {
- exec {
- executable 'sh'
- args '-c', ". ${envdir}/bin/activate && pip install -r $requirementsFile"
- }
- }
+
doLast {
def testOpts = basicTestOpts
def argMap = [
@@ -506,7 +500,6 @@ task mockAPITests {
"runner": "TestDataflowRunner",
"project": "apache-beam-testing",
"region": "us-west1",
- "requirements_file": "$requirementsFile"
]
def cmdArgs = mapToArgString(argMap)
exec {
diff --git a/sdks/python/test-suites/direct/common.gradle b/sdks/python/test-suites/direct/common.gradle
index 084c51c9b4e..1851a4d9cd0 100644
--- a/sdks/python/test-suites/direct/common.gradle
+++ b/sdks/python/test-suites/direct/common.gradle
@@ -370,23 +370,17 @@ task transformersInferenceTest {
}
}
-// Enrichment transform tests that uses Redis
-task enrichmentRedisTest {
+// Integration tests that use testcontainers
+task testcontainersTest {
dependsOn 'installGcpTest'
dependsOn ':sdks:python:sdist'
- def requirementsFile = "${rootDir}/sdks/python/apache_beam/transforms/enrichment_handlers/vertex_ai_feature_store_tests_requirement.txt"
- doFirst {
- exec {
- executable 'sh'
- args '-c', ". ${envdir}/bin/activate && pip install -r $requirementsFile"
- }
- }
+
doLast {
def testOpts = basicTestOpts
def argMap = [
"test_opts": testOpts,
"suite": "postCommitIT-direct-py${pythonVersionSuffix}",
- "collect": "uses_redis",
+ "collect": "uses_testcontainer",
"runner": "TestDirectRunner",
"region": "us-central1",
]
@@ -406,7 +400,7 @@ project.tasks.register("inferencePostCommitIT") {
'tensorflowInferenceTest',
'xgboostInferenceTest',
'transformersInferenceTest',
- 'enrichmentRedisTest',
+ 'testcontainersTest',
// (TODO) https://github.com/apache/beam/issues/25799
// uncomment tfx bsl tests once tfx supports protobuf 4.x
// 'tfxInferenceTest',