You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by al...@apache.org on 2019/04/02 17:09:38 UTC
[beam] branch master updated: [BEAM-6619] [BEAM-6593] [BEAM-6711]
Add bigquery integration tests to postcommit
This is an automated email from the ASF dual-hosted git repository.
altay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 9dfd935 [BEAM-6619] [BEAM-6593] [BEAM-6711] Add bigquery integration tests to postcommit
new b6800ce Merge pull request #8076 from Juta/bq-io-tests
9dfd935 is described below
commit 9dfd9358bd89ef963e30487c15dbf8f7f40d8189
Author: Juta <ju...@gmail.com>
AuthorDate: Mon Mar 18 09:46:36 2019 +0100
[BEAM-6619] [BEAM-6593] [BEAM-6711] Add bigquery integration tests to postcommit
---
sdks/python/apache_beam/examples/complete/game/game_stats.py | 1 +
.../apache_beam/examples/complete/game/game_stats_it_test.py | 9 ++-------
.../examples/complete/game/hourly_team_score_it_test.py | 6 ------
sdks/python/apache_beam/examples/complete/game/leader_board.py | 1 +
.../apache_beam/examples/complete/game/leader_board_it_test.py | 9 ++-------
5 files changed, 6 insertions(+), 20 deletions(-)
diff --git a/sdks/python/apache_beam/examples/complete/game/game_stats.py b/sdks/python/apache_beam/examples/complete/game/game_stats.py
index 5f7a0ed..9b5cc32 100644
--- a/sdks/python/apache_beam/examples/complete/game/game_stats.py
+++ b/sdks/python/apache_beam/examples/complete/game/game_stats.py
@@ -306,6 +306,7 @@ def run(argv=None):
topic=args.topic)
raw_events = (
scores
+ | 'DecodeString' >> beam.Map(lambda b: b.decode('utf-8'))
| 'ParseGameEventFn' >> beam.ParDo(ParseGameEventFn())
| 'AddEventTimestamps' >> beam.Map(
lambda elem: beam.window.TimestampedValue(elem, elem['timestamp'])))
diff --git a/sdks/python/apache_beam/examples/complete/game/game_stats_it_test.py b/sdks/python/apache_beam/examples/complete/game/game_stats_it_test.py
index e7b89aa..cba4b00 100644
--- a/sdks/python/apache_beam/examples/complete/game/game_stats_it_test.py
+++ b/sdks/python/apache_beam/examples/complete/game/game_stats_it_test.py
@@ -33,8 +33,6 @@ Usage:
from __future__ import absolute_import
import logging
-import os
-import sys
import time
import unittest
import uuid
@@ -51,10 +49,6 @@ from apache_beam.testing.pipeline_verifiers import PipelineStateMatcher
from apache_beam.testing.test_pipeline import TestPipeline
-@unittest.skipIf(sys.version_info[0] == 3 and
- os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
- 'This test still needs to be fixed on Python 3'
- 'TODO: BEAM-6711')
class GameStatsIT(unittest.TestCase):
# Input events containing user, team, score, processing time, window start.
@@ -102,7 +96,8 @@ class GameStatsIT(unittest.TestCase):
for _ in range(message_count):
self.pub_client.publish(topic.name,
- self.INPUT_EVENT % self._test_timestamp)
+ (self.INPUT_EVENT % self._test_timestamp
+ ).encode('utf-8'))
def _cleanup_pubsub(self):
test_utils.cleanup_subscriptions(self.sub_client, [self.input_sub])
diff --git a/sdks/python/apache_beam/examples/complete/game/hourly_team_score_it_test.py b/sdks/python/apache_beam/examples/complete/game/hourly_team_score_it_test.py
index 2fce1fc..5685132 100644
--- a/sdks/python/apache_beam/examples/complete/game/hourly_team_score_it_test.py
+++ b/sdks/python/apache_beam/examples/complete/game/hourly_team_score_it_test.py
@@ -33,8 +33,6 @@ Usage:
from __future__ import absolute_import
import logging
-import os
-import sys
import unittest
from hamcrest.core.core.allof import all_of
@@ -48,10 +46,6 @@ from apache_beam.testing.pipeline_verifiers import PipelineStateMatcher
from apache_beam.testing.test_pipeline import TestPipeline
-@unittest.skipIf(sys.version_info[0] == 3 and
- os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
- 'This test still needs to be fixed on Python 3'
- 'TODO: BEAM-6870')
class HourlyTeamScoreIT(unittest.TestCase):
DEFAULT_INPUT_FILE = 'gs://dataflow-samples/game/gaming_data*'
diff --git a/sdks/python/apache_beam/examples/complete/game/leader_board.py b/sdks/python/apache_beam/examples/complete/game/leader_board.py
index cde1544..43a599b 100644
--- a/sdks/python/apache_beam/examples/complete/game/leader_board.py
+++ b/sdks/python/apache_beam/examples/complete/game/leader_board.py
@@ -315,6 +315,7 @@ def run(argv=None):
events = (
scores
+ | 'DecodeString' >> beam.Map(lambda b: b.decode('utf-8'))
| 'ParseGameEventFn' >> beam.ParDo(ParseGameEventFn())
| 'AddEventTimestamps' >> beam.Map(
lambda elem: beam.window.TimestampedValue(elem, elem['timestamp'])))
diff --git a/sdks/python/apache_beam/examples/complete/game/leader_board_it_test.py b/sdks/python/apache_beam/examples/complete/game/leader_board_it_test.py
index b86e49e..9f057fd 100644
--- a/sdks/python/apache_beam/examples/complete/game/leader_board_it_test.py
+++ b/sdks/python/apache_beam/examples/complete/game/leader_board_it_test.py
@@ -33,8 +33,6 @@ Usage:
from __future__ import absolute_import
import logging
-import os
-import sys
import time
import unittest
import uuid
@@ -52,10 +50,6 @@ from apache_beam.testing.pipeline_verifiers import PipelineStateMatcher
from apache_beam.testing.test_pipeline import TestPipeline
-@unittest.skipIf(sys.version_info[0] == 3 and
- os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
- 'This test still needs to be fixed on Python 3'
- 'TODO: BEAM-6711')
class LeaderBoardIT(unittest.TestCase):
# Input event containing user, team, score, processing time, window start.
@@ -104,7 +98,8 @@ class LeaderBoardIT(unittest.TestCase):
for _ in range(message_count):
self.pub_client.publish(topic.name,
- self.INPUT_EVENT % self._test_timestamp)
+ (self.INPUT_EVENT % self._test_timestamp
+ ).encode('utf-8'))
def _cleanup_pubsub(self):
test_utils.cleanup_subscriptions(self.sub_client, [self.input_sub])