You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ma...@apache.org on 2019/03/26 18:12:25 UTC
[beam] branch master updated: [BEAM-6619] [BEAM-6593] update gradle
to include all py3 it tests
This is an automated email from the ASF dual-hosted git repository.
markliu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 4db0f94 [BEAM-6619] [BEAM-6593] update gradle to include all py3 it tests
new 5521415 Merge pull request #8098 from Juta/it-tests
4db0f94 is described below
commit 4db0f94b7a65c796420ed6160ae0e2bd9c415d98
Author: Juta <ju...@gmail.com>
AuthorDate: Wed Mar 20 14:06:40 2019 +0100
[BEAM-6619] [BEAM-6593] update gradle to include all py3 it tests
---
.../examples/complete/game/game_stats_it_test.py | 6 ++++++
.../complete/game/hourly_team_score_it_test.py | 6 ++++++
.../examples/complete/game/leader_board_it_test.py | 6 ++++++
sdks/python/apache_beam/examples/fastavro_it_test.py | 6 ++++++
.../io/gcp/big_query_query_to_table_it_test.py | 6 ++++++
.../apache_beam/io/gcp/bigquery_file_loads_test.py | 5 +++++
sdks/python/test-suites/dataflow/py3/build.gradle | 18 +-----------------
sdks/python/test-suites/direct/py3/build.gradle | 16 +++-------------
8 files changed, 39 insertions(+), 30 deletions(-)
diff --git a/sdks/python/apache_beam/examples/complete/game/game_stats_it_test.py b/sdks/python/apache_beam/examples/complete/game/game_stats_it_test.py
index 67b80c0..e7b89aa 100644
--- a/sdks/python/apache_beam/examples/complete/game/game_stats_it_test.py
+++ b/sdks/python/apache_beam/examples/complete/game/game_stats_it_test.py
@@ -33,6 +33,8 @@ Usage:
from __future__ import absolute_import
import logging
+import os
+import sys
import time
import unittest
import uuid
@@ -49,6 +51,10 @@ from apache_beam.testing.pipeline_verifiers import PipelineStateMatcher
from apache_beam.testing.test_pipeline import TestPipeline
+@unittest.skipIf(sys.version_info[0] == 3 and
+ os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
+ 'This test still needs to be fixed on Python 3'
+ 'TODO: BEAM-6711')
class GameStatsIT(unittest.TestCase):
# Input events containing user, team, score, processing time, window start.
diff --git a/sdks/python/apache_beam/examples/complete/game/hourly_team_score_it_test.py b/sdks/python/apache_beam/examples/complete/game/hourly_team_score_it_test.py
index 5685132..2fce1fc 100644
--- a/sdks/python/apache_beam/examples/complete/game/hourly_team_score_it_test.py
+++ b/sdks/python/apache_beam/examples/complete/game/hourly_team_score_it_test.py
@@ -33,6 +33,8 @@ Usage:
from __future__ import absolute_import
import logging
+import os
+import sys
import unittest
from hamcrest.core.core.allof import all_of
@@ -46,6 +48,10 @@ from apache_beam.testing.pipeline_verifiers import PipelineStateMatcher
from apache_beam.testing.test_pipeline import TestPipeline
+@unittest.skipIf(sys.version_info[0] == 3 and
+ os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
+ 'This test still needs to be fixed on Python 3'
+ 'TODO: BEAM-6870')
class HourlyTeamScoreIT(unittest.TestCase):
DEFAULT_INPUT_FILE = 'gs://dataflow-samples/game/gaming_data*'
diff --git a/sdks/python/apache_beam/examples/complete/game/leader_board_it_test.py b/sdks/python/apache_beam/examples/complete/game/leader_board_it_test.py
index 27a170f..b86e49e 100644
--- a/sdks/python/apache_beam/examples/complete/game/leader_board_it_test.py
+++ b/sdks/python/apache_beam/examples/complete/game/leader_board_it_test.py
@@ -33,6 +33,8 @@ Usage:
from __future__ import absolute_import
import logging
+import os
+import sys
import time
import unittest
import uuid
@@ -50,6 +52,10 @@ from apache_beam.testing.pipeline_verifiers import PipelineStateMatcher
from apache_beam.testing.test_pipeline import TestPipeline
+@unittest.skipIf(sys.version_info[0] == 3 and
+ os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
+ 'This test still needs to be fixed on Python 3'
+ 'TODO: BEAM-6711')
class LeaderBoardIT(unittest.TestCase):
# Input event containing user, team, score, processing time, window start.
diff --git a/sdks/python/apache_beam/examples/fastavro_it_test.py b/sdks/python/apache_beam/examples/fastavro_it_test.py
index 327b10f..61833eb 100644
--- a/sdks/python/apache_beam/examples/fastavro_it_test.py
+++ b/sdks/python/apache_beam/examples/fastavro_it_test.py
@@ -46,6 +46,8 @@ from __future__ import absolute_import
from __future__ import division
import logging
+import os
+import sys
import unittest
import uuid
@@ -82,6 +84,10 @@ def record(i):
}
+@unittest.skipIf(sys.version_info[0] == 3 and
+ os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
+ 'This test still needs to be fixed on Python 3'
+ 'TODO: BEAM-6522')
class FastavroIT(unittest.TestCase):
SCHEMA = Parse('''
diff --git a/sdks/python/apache_beam/io/gcp/big_query_query_to_table_it_test.py b/sdks/python/apache_beam/io/gcp/big_query_query_to_table_it_test.py
index 1a1b401..2570fc7 100644
--- a/sdks/python/apache_beam/io/gcp/big_query_query_to_table_it_test.py
+++ b/sdks/python/apache_beam/io/gcp/big_query_query_to_table_it_test.py
@@ -22,7 +22,9 @@ from __future__ import absolute_import
import datetime
import logging
+import os
import random
+import sys
import time
import unittest
@@ -182,6 +184,10 @@ class BigQueryQueryToTableIT(unittest.TestCase):
self.project, self.dataset_id, 'output_table')
self.assertEqual(KMS_KEY, table.encryptionConfiguration.kmsKeyName)
+ @unittest.skipIf(sys.version_info[0] == 3 and
+ os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
+ 'This test still needs to be fixed on Python 3'
+ 'TODO: BEAM-6769')
@attr('IT')
def test_big_query_new_types(self):
expected_checksum = test_utils.compute_hash(NEW_TYPES_OUTPUT_EXPECTED)
diff --git a/sdks/python/apache_beam/io/gcp/bigquery_file_loads_test.py b/sdks/python/apache_beam/io/gcp/bigquery_file_loads_test.py
index 44d378b..499f0c3 100644
--- a/sdks/python/apache_beam/io/gcp/bigquery_file_loads_test.py
+++ b/sdks/python/apache_beam/io/gcp/bigquery_file_loads_test.py
@@ -23,6 +23,7 @@ import json
import logging
import os
import random
+import sys
import time
import unittest
@@ -360,6 +361,10 @@ class BigQueryFileLoadsIT(unittest.TestCase):
logging.info("Created dataset %s in project %s",
self.dataset_id, self.project)
+ @unittest.skipIf(sys.version_info[0] == 3 and
+ os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
+ 'This test still needs to be fixed on Python 3'
+ 'TODO: BEAM-6711')
@attr('IT')
def test_multiple_destinations_transform(self):
output_table_1 = '%s%s' % (self.output_table, 1)
diff --git a/sdks/python/test-suites/dataflow/py3/build.gradle b/sdks/python/test-suites/dataflow/py3/build.gradle
index b033153..15e2a49 100644
--- a/sdks/python/test-suites/dataflow/py3/build.gradle
+++ b/sdks/python/test-suites/dataflow/py3/build.gradle
@@ -33,23 +33,7 @@ def basicTestOpts = [
task postCommitIT(dependsOn: ['sdist', 'installGcpTest']) {
doLast {
- def tests = [
- "apache_beam.examples.wordcount_it_test:WordCountIT.test_wordcount_it",
- "apache_beam.examples.cookbook.bigquery_tornadoes_it_test:BigqueryTornadoesIT.test_bigquery_tornadoes_it",
- "apache_beam.examples.streaming_wordcount_it_test:StreamingWordCountIT.test_streaming_wordcount_it",
- "apache_beam.examples.wordcount_it_test:WordCountIT.test_wordcount_fnapi_it",
- "apache_beam.io.gcp.gcsio_integration_test:GcsIOIntegrationTest.test_copy",
- "apache_beam.io.gcp.gcsio_integration_test:GcsIOIntegrationTest.test_copy_batch",
- "apache_beam.io.gcp.gcsio_integration_test:GcsIOIntegrationTest.test_copy_batch_kms",
- "apache_beam.io.gcp.gcsio_integration_test:GcsIOIntegrationTest.test_copy_batch_rewrite_token",
- "apache_beam.io.gcp.gcsio_integration_test:GcsIOIntegrationTest.test_copy_kms",
- "apache_beam.io.gcp.gcsio_integration_test:GcsIOIntegrationTest.test_copy_rewrite_token",
- "apache_beam.io.gcp.pubsub_integration_test:PubSubIntegrationTest.test_streaming_data_only",
- "apache_beam.io.gcp.pubsub_integration_test:PubSubIntegrationTest.test_streaming_with_attributes",
- "apache_beam.io.parquetio_it_test:TestParquetIT.test_parquetio_it",
- "apache_beam.examples.complete.autocomplete_test:AutocompleteTest.test_autocomplete_it",
- ]
- def testOpts = basicTestOpts + ["--tests=${tests.join(',')}"]
+ def testOpts = basicTestOpts + ["--attr=IT"]
def cmdArgs = project.mapToArgString([
"test_opts": testOpts,
"sdk_location": "${project.buildDir}/apache-beam.tar.gz"
diff --git a/sdks/python/test-suites/direct/py3/build.gradle b/sdks/python/test-suites/direct/py3/build.gradle
index e1b358a..a1321ee 100644
--- a/sdks/python/test-suites/direct/py3/build.gradle
+++ b/sdks/python/test-suites/direct/py3/build.gradle
@@ -29,19 +29,9 @@ task postCommitIT(dependsOn: 'installGcpTest') {
doLast {
def batchTests = [
"apache_beam.examples.wordcount_it_test:WordCountIT.test_wordcount_it",
- "apache_beam.examples.cookbook.bigquery_tornadoes_it_test:BigqueryTornadoesIT.test_bigquery_tornadoes_it",
- "apache_beam.examples.streaming_wordcount_it_test:StreamingWordCountIT.test_streaming_wordcount_it",
- "apache_beam.examples.wordcount_it_test:WordCountIT.test_wordcount_fnapi_it",
- "apache_beam.io.gcp.gcsio_integration_test:GcsIOIntegrationTest.test_copy",
- "apache_beam.io.gcp.gcsio_integration_test:GcsIOIntegrationTest.test_copy_batch",
- "apache_beam.io.gcp.gcsio_integration_test:GcsIOIntegrationTest.test_copy_batch_kms",
- "apache_beam.io.gcp.gcsio_integration_test:GcsIOIntegrationTest.test_copy_batch_rewrite_token",
- "apache_beam.io.gcp.gcsio_integration_test:GcsIOIntegrationTest.test_copy_kms",
- "apache_beam.io.gcp.gcsio_integration_test:GcsIOIntegrationTest.test_copy_rewrite_token",
- "apache_beam.io.gcp.pubsub_integration_test:PubSubIntegrationTest.test_streaming_data_only",
- "apache_beam.io.gcp.pubsub_integration_test:PubSubIntegrationTest.test_streaming_with_attributes",
- "apache_beam.io.parquetio_it_test:TestParquetIT.test_parquetio_it",
- "apache_beam.examples.complete.autocomplete_test:AutocompleteTest.test_autocomplete_it",
+ "apache_beam.io.gcp.pubsub_integration_test:PubSubIntegrationTest",
+ "apache_beam.io.gcp.big_query_query_to_table_it_test:BigQueryQueryToTableIT",
+ "apache_beam.io.gcp.bigquery_io_read_it_test",
]
def testOpts = [
"--tests=${batchTests.join(',')}",