You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ud...@apache.org on 2021/08/18 20:59:30 UTC
[beam] branch master updated: [BEAM-3713] Cleanup,
remove nosetest references (#15245)
This is an automated email from the ASF dual-hosted git repository.
udim pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 9188bbf [BEAM-3713] Cleanup, remove nosetest references (#15245)
9188bbf is described below
commit 9188bbf05c4bdd0d306257d1c38c501c61398577
Author: Benjamin Gonzalez <74...@users.noreply.github.com>
AuthorDate: Wed Aug 18 15:58:26 2021 -0500
[BEAM-3713] Cleanup, remove nosetest references (#15245)
* [BEAM-3713] Fix lint
* [BEAM-3713] Remove --pytest param because is no longer required
Co-authored-by: Udi Meiri <ud...@users.noreply.github.com>
---
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 2 -
.../cookbook/bigquery_tornadoes_it_test.py | 3 --
sdks/python/apache_beam/testing/test_pipeline.py | 4 +-
.../transforms/periodicsequence_test.py | 2 -
.../apache_beam/transforms/ptransform_test.py | 3 --
.../apache_beam/transforms/sideinputs_test.py | 3 --
.../transforms_keyword_only_args_test.py | 4 --
sdks/python/container/base_image_requirements.txt | 2 -
sdks/python/scripts/run_integration_test.sh | 45 +++++++-------------
sdks/python/setup.py | 7 ----
sdks/python/test-suites/dataflow/common.gradle | 5 ---
sdks/python/test-suites/direct/common.gradle | 3 --
sdks/python/test-suites/portable/common.gradle | 5 +--
sdks/python/test_config.py | 49 ----------------------
14 files changed, 18 insertions(+), 119 deletions(-)
diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
index f5a3c19..ce507db 100644
--- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
+++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
@@ -2174,7 +2174,6 @@ class BeamModulePlugin implements Plugin<Project> {
"pipeline_opts": config.pythonPipelineOptions + sdkLocationOpt,
"test_opts": config.pytestOptions,
"suite": "xlangValidateRunner",
- "pytest": true, // TODO(BEAM-3713): Remove this once nose is removed.
"collect": config.pythonTestAttr
]
def cmdArgs = project.project(':sdks:python').mapToArgString(beamPythonTestPipelineOptions)
@@ -2221,7 +2220,6 @@ class BeamModulePlugin implements Plugin<Project> {
"pipeline_opts": config.pythonPipelineOptions + sdkLocationOpt,
"test_opts": config.pytestOptions,
"suite": "xlangSqlValidateRunner",
- "pytest": true, // TODO(BEAM-3713): Remove this once nose is removed.
"collect": "xlang_sql_expansion_service"
]
def cmdArgs = project.project(':sdks:python').mapToArgString(beamPythonTestPipelineOptions)
diff --git a/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py b/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py
index fa5f12c..8d44615 100644
--- a/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py
+++ b/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py
@@ -35,9 +35,6 @@ from apache_beam.testing.test_pipeline import TestPipeline
class BigqueryTornadoesIT(unittest.TestCase):
- # Enable nose tests running in parallel
- _multiprocess_can_split_ = True
-
# The default checksum is a SHA-1 hash generated from sorted rows reading
# from expected Bigquery table.
DEFAULT_CHECKSUM = 'd860e636050c559a16a791aff40d6ad809d4daf0'
diff --git a/sdks/python/apache_beam/testing/test_pipeline.py b/sdks/python/apache_beam/testing/test_pipeline.py
index 2ba273e..910f149 100644
--- a/sdks/python/apache_beam/testing/test_pipeline.py
+++ b/sdks/python/apache_beam/testing/test_pipeline.py
@@ -40,7 +40,7 @@ class TestPipeline(Pipeline):
It has a functionality to parse arguments from command line and build pipeline
options for tests who runs against a pipeline runner and utilizes resources
of the pipeline runner. Those test functions are recommended to be tagged by
- ``@attr("ValidatesRunner")`` annotation.
+ ``@pytest.mark.it_validatesrunner`` annotation.
In order to configure the test with customized pipeline options from command
line, system argument ``--test-pipeline-options`` can be used to obtains a
@@ -48,7 +48,7 @@ class TestPipeline(Pipeline):
For example, use following command line to execute all ValidatesRunner tests::
- python setup.py nosetests -a ValidatesRunner \\
+ pytest -m it_validatesrunner \\
--test-pipeline-options="--runner=DirectRunner \\
--job_name=myJobName \\
--num_workers=1"
diff --git a/sdks/python/apache_beam/transforms/periodicsequence_test.py b/sdks/python/apache_beam/transforms/periodicsequence_test.py
index 7e6549e..b18bf75 100644
--- a/sdks/python/apache_beam/transforms/periodicsequence_test.py
+++ b/sdks/python/apache_beam/transforms/periodicsequence_test.py
@@ -35,8 +35,6 @@ from apache_beam.transforms.periodicsequence import PeriodicSequence
class PeriodicSequenceTest(unittest.TestCase):
- # Enable nose tests running in parallel
-
def test_periodicsequence_outputs_valid_sequence(self):
start_offset = 1
start_time = time.time() + start_offset
diff --git a/sdks/python/apache_beam/transforms/ptransform_test.py b/sdks/python/apache_beam/transforms/ptransform_test.py
index beb7a62..ec001e2 100644
--- a/sdks/python/apache_beam/transforms/ptransform_test.py
+++ b/sdks/python/apache_beam/transforms/ptransform_test.py
@@ -67,9 +67,6 @@ from apache_beam.utils.windowed_value import WindowedValue
class PTransformTest(unittest.TestCase):
- # Enable nose tests running in parallel
- _multiprocess_can_split_ = True
-
def assertStartswith(self, msg, prefix):
self.assertTrue(
msg.startswith(prefix), '"%s" does not start with "%s"' % (msg, prefix))
diff --git a/sdks/python/apache_beam/transforms/sideinputs_test.py b/sdks/python/apache_beam/transforms/sideinputs_test.py
index 391ecdc..29a8845 100644
--- a/sdks/python/apache_beam/transforms/sideinputs_test.py
+++ b/sdks/python/apache_beam/transforms/sideinputs_test.py
@@ -38,9 +38,6 @@ from apache_beam.utils.timestamp import Timestamp
class SideInputsTest(unittest.TestCase):
- # Enable nose tests running in parallel
- _multiprocess_can_split_ = True
-
def create_pipeline(self):
return TestPipeline()
diff --git a/sdks/python/apache_beam/transforms/transforms_keyword_only_args_test.py b/sdks/python/apache_beam/transforms/transforms_keyword_only_args_test.py
index 80e0aff..28566ba 100644
--- a/sdks/python/apache_beam/transforms/transforms_keyword_only_args_test.py
+++ b/sdks/python/apache_beam/transforms/transforms_keyword_only_args_test.py
@@ -29,10 +29,6 @@ from apache_beam.testing.util import equal_to
class KeywordOnlyArgsTests(unittest.TestCase):
-
- # Enable nose tests running in parallel
- _multiprocess_can_split_ = True
-
def test_side_input_keyword_only_args(self):
with TestPipeline() as pipeline:
diff --git a/sdks/python/container/base_image_requirements.txt b/sdks/python/container/base_image_requirements.txt
index c627b4e..6bfebfb 100644
--- a/sdks/python/container/base_image_requirements.txt
+++ b/sdks/python/container/base_image_requirements.txt
@@ -87,5 +87,3 @@ tensorflow==2.5.0
tenacity>=5.0.2
pyhamcrest<2.0,>=1.9
pytest==4.6.11
-# TODO(BEAM-3713): Remove once nose is no longer in use.
-nose==1.3.7
diff --git a/sdks/python/scripts/run_integration_test.sh b/sdks/python/scripts/run_integration_test.sh
index c5baa89..b477e91 100755
--- a/sdks/python/scripts/run_integration_test.sh
+++ b/sdks/python/scripts/run_integration_test.sh
@@ -45,10 +45,10 @@
# using this flag.
#
# Test related flags:
-# test_opts -> List of space separated options to configure Nose test
-# during execution. Commonly used options like `--attr`,
-# `--tests`, `--nologcapture`. More can be found in
-# https://nose.readthedocs.io/en/latest/man.html#options
+# test_opts -> List of space separated options to configure Pytest test
+# during execution. Commonly used options like `--capture=no`
+# `--collect-only`. More can be found in
+# https://docs.pytest.org/en/latest/reference.html#command-line-flags
# suite -> Namespace for this run of tests. Required if running
# under Jenkins. Used to differentiate runs of the same
# tests with different interpreters/dependencies/etc.
@@ -58,7 +58,7 @@
# `$ ./run_integration_test.sh`
#
# - Run single integration test with default pipeline options:
-# `$ ./run_integration_test.sh --test_opts --tests=apache_beam.examples.wordcount_it_test:WordCountIT.test_wordcount_it`
+# `$ ./run_integration_test.sh --test_opts apache_beam/examples/wordcount_it_test.py::WordCountIT::test_wordcount_it`
#
# - Run full set of PostCommit tests with customized pipeline options:
# `$ ./run_integration_test.sh --project my-project --gcs_location gs://my-location`
@@ -80,10 +80,10 @@ KMS_KEY_NAME="projects/apache-beam-testing/locations/global/keyRings/beam-it/cry
SUITE=""
COLLECT_MARKERS=
-# Default test (nose) options.
+# Default test (pytest) options.
# Run WordCountIT.test_wordcount_it by default if no test options are
# provided.
-TEST_OPTS="--tests=apache_beam.examples.wordcount_it_test:WordCountIT.test_wordcount_it --nocapture"
+TEST_OPTS="apache_beam/examples/wordcount_it_test.py::WordCountIT::test_wordcount_it"
while [[ $# -gt 0 ]]
do
@@ -164,11 +164,6 @@ case $key in
shift # past argument
shift # past value
;;
- --pytest)
- PYTEST="$2"
- shift # past argument
- shift # past value
- ;;
--collect)
COLLECT_MARKERS="-m=$2"
shift # past argument
@@ -185,7 +180,6 @@ if [[ "$JENKINS_HOME" != "" && "$SUITE" == "" ]]; then
echo "Argument --suite is required in a Jenkins environment."
exit 1
fi
-XUNIT_FILE="nosetests-$SUITE.xml"
set -o errexit
@@ -281,23 +275,12 @@ fi
# Run tests and validate that jobs finish successfully.
echo ">>> RUNNING integration tests with pipeline options: $PIPELINE_OPTS"
-if [[ "$PYTEST" = true ]]; then
- echo ">>> pytest options: $TEST_OPTS"
- echo ">>> collect markers: $COLLECT_MARKERS"
- ARGS="-o junit_suite_name=$SUITE --junitxml=pytest_$SUITE.xml $TEST_OPTS"
- # Handle markers as an independient argument from $TEST_OPTS to prevent errors in space separeted flags
- if [ -z "$COLLECT_MARKERS" ]; then
- pytest $ARGS --test-pipeline-options="$PIPELINE_OPTS"
- else
- pytest $ARGS --test-pipeline-options="$PIPELINE_OPTS" "$COLLECT_MARKERS"
- fi
+echo ">>> pytest options: $TEST_OPTS"
+echo ">>> collect markers: $COLLECT_MARKERS"
+ARGS="-o junit_suite_name=$SUITE --junitxml=pytest_$SUITE.xml $TEST_OPTS"
+# Handle markers as an independient argument from $TEST_OPTS to prevent errors in space separeted flags
+if [ -z "$COLLECT_MARKERS" ]; then
+ pytest $ARGS --test-pipeline-options="$PIPELINE_OPTS"
else
- echo ">>> test options: $TEST_OPTS"
- # TODO(BEAM-3713): Pass $SUITE once migrated to pytest. xunitmp doesn't
- # support suite names.
- python setup.py nosetests \
- --test-pipeline-options="$PIPELINE_OPTS" \
- --with-xunitmp --xunitmp-file=$XUNIT_FILE \
- --ignore-files '.*py3\d?\.py$' \
- $TEST_OPTS
+ pytest $ARGS --test-pipeline-options="$PIPELINE_OPTS" "$COLLECT_MARKERS"
fi
\ No newline at end of file
diff --git a/sdks/python/setup.py b/sdks/python/setup.py
index 92d001a..5d72a9d 100644
--- a/sdks/python/setup.py
+++ b/sdks/python/setup.py
@@ -166,8 +166,6 @@ if sys.platform == 'win32' and sys.maxsize <= 2**32:
REQUIRED_TEST_PACKAGES = [
'freezegun>=0.3.12',
'mock>=1.0.1,<3.0.0',
- 'nose>=1.3.7',
- 'nose_xunitmp>=0.4.1',
'pandas>=1.0,<1.4.0',
'parameterized>=0.7.1,<0.8.0',
'pyhamcrest>=1.9,!=1.10.0,<2.0.0',
@@ -290,7 +288,6 @@ setuptools.setup(
]),
install_requires=REQUIRED_PACKAGES,
python_requires=python_requires,
- test_suite='nose.collector',
# BEAM-8840: Do NOT use tests_require or setup_requires.
extras_require={
'docs': ['Sphinx>=1.5.2,<2.0'],
@@ -317,10 +314,6 @@ setuptools.setup(
],
license='Apache License, Version 2.0',
keywords=PACKAGE_KEYWORDS,
- entry_points={
- 'nose.plugins.0.10': [
- 'beam_test_plugin = test_config:BeamTestPlugin',
- ]},
cmdclass={
'build_py': generate_protos_first(build_py),
'develop': generate_protos_first(develop),
diff --git a/sdks/python/test-suites/dataflow/common.gradle b/sdks/python/test-suites/dataflow/common.gradle
index 7d1cf2b..29373b5 100644
--- a/sdks/python/test-suites/dataflow/common.gradle
+++ b/sdks/python/test-suites/dataflow/common.gradle
@@ -65,7 +65,6 @@ def preCommitIT(String runScriptsDir, String envdir, Boolean streaming, Boolean
"sdk_location": files(configurations.distTarBall.files).singleFile,
"worker_jar" : dataflowWorkerJar,
"suite" : "preCommitIT-df${pythonSuffix}",
- "pytest" : true, // TODO(BEAM-3713): Remove this once nose is removed.
]
if (runnerV2) {
@@ -115,7 +114,6 @@ task postCommitIT {
"sdk_location": files(configurations.distTarBall.files).singleFile,
"worker_jar": dataflowWorkerJar,
"suite": "postCommitIT-df${pythonVersionSuffix}",
- "pytest": true, // TODO(BEAM-3713): Remove this once nose is removed.
"collect": "it_postcommit"
]
def cmdArgs = mapToArgString(argMap)
@@ -137,7 +135,6 @@ task validatesRunnerBatchTests {
"worker_jar" : dataflowWorkerJar,
"sdk_location": files(configurations.distTarBall.files).singleFile,
"suite" : "validatesRunnerBatchTests-df${pythonVersionSuffix}",
- "pytest": true, // TODO(BEAM-3713): Remove this once nose is removed.
"collect": "it_validatesrunner and not no_sickbay_batch"
]
@@ -172,7 +169,6 @@ task validatesRunnerStreamingTests {
"sdk_location": files(configurations.distTarBall.files).singleFile,
"worker_jar": dataflowWorkerJar,
"suite": "validatesRunnerStreamingTests-df${pythonVersionSuffix}-xdist",
- "pytest": true, // TODO(BEAM-3713): Remove this once nose is removed
"collect": "it_validatesrunner and not no_sickbay_streaming and not no_xdist"
]
if (project.hasProperty('useRunnerV2')) {
@@ -196,7 +192,6 @@ task validatesRunnerStreamingTests {
"sdk_location": files(configurations.distTarBall.files).singleFile,
"worker_jar": dataflowWorkerJar,
"suite": "validatesRunnerStreamingTests-df${pythonVersionSuffix}-noxdist",
- "pytest": true, // TODO(BEAM-3713): Remove this once nose is removed
"collect": "it_validatesrunner and not no_sickbay_streaming and no_xdist"
]
if (project.hasProperty('useRunnerV2')) {
diff --git a/sdks/python/test-suites/direct/common.gradle b/sdks/python/test-suites/direct/common.gradle
index 50fe59a..3680bfa 100644
--- a/sdks/python/test-suites/direct/common.gradle
+++ b/sdks/python/test-suites/direct/common.gradle
@@ -48,7 +48,6 @@ task postCommitIT {
def argMap = ["runner": "TestDirectRunner",
"test_opts": testOpts,
"suite": "postCommitIT-direct-py${pythonVersionSuffix}",
- "pytest": true, // TODO(BEAM-3713): Remove this once nose is removed.
]
def batchCmdArgs = mapToArgString(argMap)
exec {
@@ -108,7 +107,6 @@ task directRunnerIT {
def argMap = ["runner": "TestDirectRunner",
"test_opts": batchTestOpts,
"suite": "directRunnerIT-batch",
- "pytest": true, // TODO(BEAM-3713): Remove this once nose is removed.
]
def batchCmdArgs = mapToArgString(argMap)
exec {
@@ -131,7 +129,6 @@ task directRunnerIT {
"streaming": "true",
"test_opts": streamingTestOpts,
"suite": "directRunnerIT-streaming",
- "pytest": true, // TODO(BEAM-3713): Remove this once nose is removed.
]
def streamingCmdArgs = mapToArgString(argMap)
exec {
diff --git a/sdks/python/test-suites/portable/common.gradle b/sdks/python/test-suites/portable/common.gradle
index 20665f0..3b1654f 100644
--- a/sdks/python/test-suites/portable/common.gradle
+++ b/sdks/python/test-suites/portable/common.gradle
@@ -66,8 +66,8 @@ task flinkTriggerTranscript() {
. ${envdir}/bin/activate \\
&& cd ${pythonRootDir} \\
&& pip install -e .[test] \\
- && python setup.py nosetests \\
- --tests apache_beam.transforms.trigger_test:WeakTestStreamTranscriptTest \\
+ && pytest \\
+ apache_beam/transforms/trigger_test.py::WeakTestStreamTranscriptTest \\
--test-pipeline-options='--runner=FlinkRunner --environment_type=LOOPBACK --flink_job_server_jar=${project(":runners:flink:${latestFlinkVersion}:job-server:").shadowJar.archivePath}'
"""
}
@@ -220,7 +220,6 @@ project.task("postCommitPy${pythonVersionSuffix}IT") {
"test_opts": testOpts,
"suite": "postCommitIT-flink-py${pythonVersionSuffix}",
"pipeline_opts": pipelineOpts.join(" "),
- "pytest": true, // TODO(BEAM-3713): Remove this once nose is removed.
])
def kafkaJar = project(":sdks:java:testing:kafka-service:").buildTestKafkaServiceJar.archivePath
exec {
diff --git a/sdks/python/test_config.py b/sdks/python/test_config.py
deleted file mode 100644
index d916a7c..0000000
--- a/sdks/python/test_config.py
+++ /dev/null
@@ -1,49 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-"""Test configurations for nose
-
-This module contains nose plugin hooks that configures Beam tests which
-includes ValidatesRunner test and E2E integration test.
-
-TODO(BEAM-3713): Remove this module once nose is removed.
-"""
-
-from nose.plugins import Plugin
-
-
-class BeamTestPlugin(Plugin):
- """A nose plugin for Beam testing that registers command line options
-
- This plugin is registered through setuptools in entry_points.
- """
-
- def options(self, parser, env):
- """Add '--test-pipeline-options' and '--not_use-test-runner-api'
- to command line option to avoid unrecognized option error thrown by nose.
-
- The value of this option will be processed by TestPipeline and used to
- build customized pipeline for ValidatesRunner tests.
- """
- parser.add_option('--test-pipeline-options',
- action='store',
- type=str,
- help='providing pipeline options to run tests on runner')
- parser.add_option('--not-use-test-runner-api',
- action='store_true',
- default=False,
- help='whether not to use test-runner-api')