You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by tv...@apache.org on 2021/03/11 08:14:59 UTC
[beam] branch master updated: [BEAM-8288] remove py2 codepath from
interactive Beam (#14172)
This is an automated email from the ASF dual-hosted git repository.
tvalentyn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 3006e90 [BEAM-8288] remove py2 codepath from interactive Beam (#14172)
3006e90 is described below
commit 3006e90a28c3044ebbb24e660085d9fd039b8b83
Author: yoshiki.obata <12...@users.noreply.github.com>
AuthorDate: Thu Mar 11 17:14:23 2021 +0900
[BEAM-8288] remove py2 codepath from interactive Beam (#14172)
---
.../interactive/background_caching_job_test.py | 8 +------
.../display/pcoll_visualization_test.py | 10 +++------
.../interactive/display/pipeline_graph_test.py | 8 +------
.../runners/interactive/interactive_beam.py | 25 ++++++----------------
.../runners/interactive/interactive_beam_test.py | 8 +------
.../interactive/interactive_environment_test.py | 8 +------
.../runners/interactive/interactive_runner_test.py | 8 +------
.../interactive_environment_inspector_test.py | 8 +------
.../interactive/options/capture_control_test.py | 8 +------
.../runners/interactive/pipeline_fragment_test.py | 8 +------
.../runners/interactive/pipeline_instrument.py | 4 +---
.../runners/interactive/recording_manager_test.py | 8 +------
.../interactive/testing/integration/screen_diff.py | 10 ++-------
.../apache_beam/runners/interactive/utils_test.py | 8 +------
.../testing/test_stream_service_test.py | 8 +------
.../apache_beam/utils/interactive_utils_test.py | 8 +------
16 files changed, 25 insertions(+), 120 deletions(-)
diff --git a/sdks/python/apache_beam/runners/interactive/background_caching_job_test.py b/sdks/python/apache_beam/runners/interactive/background_caching_job_test.py
index 1388845..1600df3 100644
--- a/sdks/python/apache_beam/runners/interactive/background_caching_job_test.py
+++ b/sdks/python/apache_beam/runners/interactive/background_caching_job_test.py
@@ -21,6 +21,7 @@
from __future__ import absolute_import
import unittest
+from unittest.mock import patch
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
@@ -37,13 +38,6 @@ from apache_beam.testing.test_stream import TestStream
from apache_beam.testing.test_stream_service import TestStreamServiceController
from apache_beam.transforms.window import TimestampedValue
-# TODO(BEAM-8288): clean up the work-around of nose tests using Python2 without
-# unittest.mock module.
-try:
- from unittest.mock import patch
-except ImportError:
- from mock import patch # type: ignore[misc]
-
_FOO_PUBSUB_SUB = 'projects/test-project/subscriptions/foo'
_BAR_PUBSUB_SUB = 'projects/test-project/subscriptions/bar'
_TEST_CACHE_KEY = 'test'
diff --git a/sdks/python/apache_beam/runners/interactive/display/pcoll_visualization_test.py b/sdks/python/apache_beam/runners/interactive/display/pcoll_visualization_test.py
index b1158e2..c36c2e1 100644
--- a/sdks/python/apache_beam/runners/interactive/display/pcoll_visualization_test.py
+++ b/sdks/python/apache_beam/runners/interactive/display/pcoll_visualization_test.py
@@ -21,6 +21,9 @@
from __future__ import absolute_import
import unittest
+from unittest.mock import ANY
+from unittest.mock import PropertyMock
+from unittest.mock import patch
import pytz
@@ -37,13 +40,6 @@ from apache_beam.transforms.window import IntervalWindow
from apache_beam.utils.windowed_value import PaneInfo
from apache_beam.utils.windowed_value import PaneInfoTiming
-# TODO(BEAM-8288): clean up the work-around of nose tests using Python2 without
-# unittest.mock module.
-try:
- from unittest.mock import patch, ANY, PropertyMock
-except ImportError:
- from mock import patch, ANY, PropertyMock # type: ignore[misc]
-
try:
import timeloop
except ImportError:
diff --git a/sdks/python/apache_beam/runners/interactive/display/pipeline_graph_test.py b/sdks/python/apache_beam/runners/interactive/display/pipeline_graph_test.py
index 01d85bd..30fc761 100644
--- a/sdks/python/apache_beam/runners/interactive/display/pipeline_graph_test.py
+++ b/sdks/python/apache_beam/runners/interactive/display/pipeline_graph_test.py
@@ -21,6 +21,7 @@
from __future__ import absolute_import
import unittest
+from unittest.mock import patch
import apache_beam as beam
from apache_beam.runners.interactive import interactive_beam as ib
@@ -29,13 +30,6 @@ from apache_beam.runners.interactive import interactive_runner as ir
from apache_beam.runners.interactive.display import pipeline_graph
from apache_beam.runners.interactive.testing.mock_ipython import mock_get_ipython
-# TODO(BEAM-8288): clean up the work-around of nose tests using Python2 without
-# unittest.mock module.
-try:
- from unittest.mock import patch
-except ImportError:
- from mock import patch # type: ignore[misc]
-
# pylint: disable=range-builtin-not-iterating,unused-variable,possibly-unused-variable
# Reason:
# Disable pylint for pipelines built for testing. Not all PCollections are
diff --git a/sdks/python/apache_beam/runners/interactive/interactive_beam.py b/sdks/python/apache_beam/runners/interactive/interactive_beam.py
index d77bf52..c4f9d2b 100644
--- a/sdks/python/apache_beam/runners/interactive/interactive_beam.py
+++ b/sdks/python/apache_beam/runners/interactive/interactive_beam.py
@@ -361,12 +361,14 @@ def watch(watchable):
ie.current_env().watch(watchable)
-# TODO(BEAM-8288): Change the signature of this function to
-# `show(*pcolls, include_window_info=False, visualize_data=False)` once Python 2
-# is completely deprecated from Beam.
@progress_indicated
-def show(*pcolls, **configs):
- # type: (*Union[Dict[Any, PCollection], Iterable[PCollection], PCollection], **bool) -> None
+def show(
+ *pcolls,
+ include_window_info=False,
+ visualize_data=False,
+ n='inf',
+ duration='inf'):
+ # type: (*Union[Dict[Any, PCollection], Iterable[PCollection], PCollection], bool, bool, Union[int, str], Union[int, str]) -> None
"""Shows given PCollections in an interactive exploratory way if used within
a notebook, or prints a heading sampled data if used within an ipython shell.
@@ -451,13 +453,6 @@ def show(*pcolls, **configs):
'{} is not an apache_beam.pvalue.PCollection.'.format(pcoll))
user_pipeline = pcolls[0].pipeline
- # TODO(BEAM-8288): Remove below pops and assertion once Python 2 is
- # deprecated from Beam.
- include_window_info = configs.pop('include_window_info', False)
- visualize_data = configs.pop('visualize_data', False)
- n = configs.pop('n', 'inf')
- duration = configs.pop('duration', 'inf')
-
if isinstance(n, str):
assert n == 'inf', (
'Currently only the string \'inf\' is supported. This denotes reading '
@@ -475,12 +470,6 @@ def show(*pcolls, **configs):
if duration == 'inf':
duration = float('inf')
- # This assertion is to protect the backward compatibility for function
- # signature change after Python 2 deprecation.
- assert not configs, (
- 'The only supported arguments are include_window_info, visualize_data, '
- 'n, and duration')
-
recording_manager = ie.current_env().get_recording_manager(
user_pipeline, create_if_absent=True)
recording = recording_manager.record(pcolls, max_n=n, max_duration=duration)
diff --git a/sdks/python/apache_beam/runners/interactive/interactive_beam_test.py b/sdks/python/apache_beam/runners/interactive/interactive_beam_test.py
index 375bca2..7733e72 100644
--- a/sdks/python/apache_beam/runners/interactive/interactive_beam_test.py
+++ b/sdks/python/apache_beam/runners/interactive/interactive_beam_test.py
@@ -24,6 +24,7 @@ import importlib
import sys
import time
import unittest
+from unittest.mock import patch
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
@@ -34,13 +35,6 @@ from apache_beam.runners.interactive.options.capture_limiters import Limiter
from apache_beam.runners.runner import PipelineState
from apache_beam.testing.test_stream import TestStream
-# TODO(BEAM-8288): clean up the work-around of nose tests using Python2 without
-# unittest.mock module.
-try:
- from unittest.mock import patch
-except ImportError:
- from mock import patch
-
# The module name is also a variable in module.
_module_name = 'apache_beam.runners.interactive.interactive_beam_test'
diff --git a/sdks/python/apache_beam/runners/interactive/interactive_environment_test.py b/sdks/python/apache_beam/runners/interactive/interactive_environment_test.py
index 8d7217a..a2c848d 100644
--- a/sdks/python/apache_beam/runners/interactive/interactive_environment_test.py
+++ b/sdks/python/apache_beam/runners/interactive/interactive_environment_test.py
@@ -22,6 +22,7 @@ from __future__ import absolute_import
import importlib
import unittest
+from unittest.mock import patch
import apache_beam as beam
from apache_beam.runners import runner
@@ -29,13 +30,6 @@ from apache_beam.runners.interactive import cache_manager as cache
from apache_beam.runners.interactive import interactive_environment as ie
from apache_beam.runners.interactive.recording_manager import RecordingManager
-# TODO(BEAM-8288): clean up the work-around of nose tests using Python2 without
-# unittest.mock module.
-try:
- from unittest.mock import patch
-except ImportError:
- from mock import patch # type: ignore[misc]
-
# The module name is also a variable in module.
_module_name = 'apache_beam.runners.interactive.interactive_environment_test'
diff --git a/sdks/python/apache_beam/runners/interactive/interactive_runner_test.py b/sdks/python/apache_beam/runners/interactive/interactive_runner_test.py
index 3fb386d..8545fea 100644
--- a/sdks/python/apache_beam/runners/interactive/interactive_runner_test.py
+++ b/sdks/python/apache_beam/runners/interactive/interactive_runner_test.py
@@ -28,6 +28,7 @@ from __future__ import print_function
import sys
import unittest
+from unittest.mock import patch
import pandas as pd
@@ -46,13 +47,6 @@ from apache_beam.utils.windowed_value import PaneInfo
from apache_beam.utils.windowed_value import PaneInfoTiming
from apache_beam.utils.windowed_value import WindowedValue
-# TODO(BEAM-8288): clean up the work-around of nose tests using Python2 without
-# unittest.mock module.
-try:
- from unittest.mock import patch
-except ImportError:
- from mock import patch # type: ignore[misc]
-
def print_with_message(msg):
def printer(elem):
diff --git a/sdks/python/apache_beam/runners/interactive/messaging/interactive_environment_inspector_test.py b/sdks/python/apache_beam/runners/interactive/messaging/interactive_environment_inspector_test.py
index 764e0f1..f2f4242 100644
--- a/sdks/python/apache_beam/runners/interactive/messaging/interactive_environment_inspector_test.py
+++ b/sdks/python/apache_beam/runners/interactive/messaging/interactive_environment_inspector_test.py
@@ -23,6 +23,7 @@ from __future__ import absolute_import
import json
import sys
import unittest
+from unittest.mock import patch
import apache_beam as beam
import apache_beam.runners.interactive.messaging.interactive_environment_inspector as inspector
@@ -32,13 +33,6 @@ from apache_beam.runners.interactive import interactive_runner as ir
from apache_beam.runners.interactive.testing.mock_ipython import mock_get_ipython
from apache_beam.runners.interactive.utils import obfuscate
-# TODO(BEAM-8288): clean up the work-around of nose tests using Python2 without
-# unittest.mock module.
-try:
- from unittest.mock import patch
-except ImportError:
- from mock import patch # type: ignore[misc]
-
@unittest.skipIf(
not ie.current_env().is_interactive_ready,
diff --git a/sdks/python/apache_beam/runners/interactive/options/capture_control_test.py b/sdks/python/apache_beam/runners/interactive/options/capture_control_test.py
index d9cf553..d119eaf 100644
--- a/sdks/python/apache_beam/runners/interactive/options/capture_control_test.py
+++ b/sdks/python/apache_beam/runners/interactive/options/capture_control_test.py
@@ -22,6 +22,7 @@
from __future__ import absolute_import
import unittest
+from unittest.mock import patch
import apache_beam as beam
from apache_beam import coders
@@ -37,13 +38,6 @@ from apache_beam.runners.interactive.options import capture_control
from apache_beam.runners.interactive.options import capture_limiters
from apache_beam.testing.test_stream_service import TestStreamServiceController
-# TODO(BEAM-8288): clean up the work-around of nose tests using Python2 without
-# unittest.mock module.
-try:
- from unittest.mock import patch
-except ImportError:
- from mock import patch # type: ignore[misc]
-
def _build_an_empty_streaming_pipeline():
from apache_beam.options.pipeline_options import PipelineOptions
diff --git a/sdks/python/apache_beam/runners/interactive/pipeline_fragment_test.py b/sdks/python/apache_beam/runners/interactive/pipeline_fragment_test.py
index 4ab5801..fc13cc9 100644
--- a/sdks/python/apache_beam/runners/interactive/pipeline_fragment_test.py
+++ b/sdks/python/apache_beam/runners/interactive/pipeline_fragment_test.py
@@ -19,6 +19,7 @@
from __future__ import absolute_import
import unittest
+from unittest.mock import patch
import apache_beam as beam
from apache_beam.options.pipeline_options import StandardOptions
@@ -31,13 +32,6 @@ from apache_beam.runners.interactive.testing.pipeline_assertion import assert_pi
from apache_beam.runners.interactive.testing.pipeline_assertion import assert_pipeline_proto_equal
from apache_beam.testing.test_stream import TestStream
-# TODO(BEAM-8288): clean up the work-around of nose tests using Python2 without
-# unittest.mock module.
-try:
- from unittest.mock import patch
-except ImportError:
- from mock import patch # type: ignore[misc]
-
@unittest.skipIf(
not ie.current_env().is_interactive_ready,
diff --git a/sdks/python/apache_beam/runners/interactive/pipeline_instrument.py b/sdks/python/apache_beam/runners/interactive/pipeline_instrument.py
index f112aeb..42b2236 100644
--- a/sdks/python/apache_beam/runners/interactive/pipeline_instrument.py
+++ b/sdks/python/apache_beam/runners/interactive/pipeline_instrument.py
@@ -885,9 +885,7 @@ def cacheables(pcolls_to_pcoll_id):
cacheable_var_by_pcoll_id = {}
for watching in ie.current_env().watching():
for key, val in watching:
- # TODO(BEAM-8288): cleanup the attribute check when py2 is not supported.
- if hasattr(val, '__class__') and isinstance(val, beam.pvalue.PCollection):
- cacheable = {}
+ if isinstance(val, beam.pvalue.PCollection):
pcoll_id = pcolls_to_pcoll_id.get(str(val), None)
# It's highly possible that PCollection str is not unique across
diff --git a/sdks/python/apache_beam/runners/interactive/recording_manager_test.py b/sdks/python/apache_beam/runners/interactive/recording_manager_test.py
index 285375f..343ad6a 100644
--- a/sdks/python/apache_beam/runners/interactive/recording_manager_test.py
+++ b/sdks/python/apache_beam/runners/interactive/recording_manager_test.py
@@ -19,6 +19,7 @@ from __future__ import absolute_import
import time
import unittest
+from unittest.mock import MagicMock
import apache_beam as beam
from apache_beam import coders
@@ -42,13 +43,6 @@ from apache_beam.transforms.window import GlobalWindow
from apache_beam.utils.timestamp import MIN_TIMESTAMP
from apache_beam.utils.windowed_value import WindowedValue
-# TODO(BEAM-8288): clean up the work-around of nose tests using Python2 without
-# unittest.mock module.
-try:
- from unittest.mock import MagicMock
-except ImportError:
- from mock import MagicMock # type: ignore[misc]
-
class MockPipelineResult(beam.runners.runner.PipelineResult):
"""Mock class for controlling a PipelineResult."""
diff --git a/sdks/python/apache_beam/runners/interactive/testing/integration/screen_diff.py b/sdks/python/apache_beam/runners/interactive/testing/integration/screen_diff.py
index 57dc0a7..54b442f 100644
--- a/sdks/python/apache_beam/runners/interactive/testing/integration/screen_diff.py
+++ b/sdks/python/apache_beam/runners/interactive/testing/integration/screen_diff.py
@@ -25,20 +25,14 @@ import os
import platform
import threading
import unittest
+from http.server import HTTPServer
+from http.server import SimpleHTTPRequestHandler
import pytest
from apache_beam.runners.interactive import interactive_environment as ie
from apache_beam.runners.interactive.testing.integration import notebook_executor
-# TODO(BEAM-8288): clean up the work-around when Python2 support is deprecated.
-try:
- from http.server import SimpleHTTPRequestHandler
- from http.server import HTTPServer
-except ImportError:
- import SimpleHTTPServer as HTTPServer
- from SimpleHTTPServer import SimpleHTTPRequestHandler
-
try:
import chromedriver_binary # pylint: disable=unused-import
from needle.cases import NeedleTestCase
diff --git a/sdks/python/apache_beam/runners/interactive/utils_test.py b/sdks/python/apache_beam/runners/interactive/utils_test.py
index 32ed887..3a335bf 100644
--- a/sdks/python/apache_beam/runners/interactive/utils_test.py
+++ b/sdks/python/apache_beam/runners/interactive/utils_test.py
@@ -20,6 +20,7 @@ from __future__ import absolute_import
import json
import logging
import unittest
+from unittest.mock import patch
import numpy as np
import pandas as pd
@@ -32,13 +33,6 @@ from apache_beam.testing.test_stream import WindowedValueHolder
from apache_beam.utils.timestamp import Timestamp
from apache_beam.utils.windowed_value import WindowedValue
-# TODO(BEAM-8288): clean up the work-around of nose tests using Python2 without
-# unittest.mock module.
-try:
- from unittest.mock import patch
-except ImportError:
- from mock import patch
-
class ParseToDataframeTest(unittest.TestCase):
def test_parse_windowedvalue(self):
diff --git a/sdks/python/apache_beam/testing/test_stream_service_test.py b/sdks/python/apache_beam/testing/test_stream_service_test.py
index 7a5b403..2c6cf5a 100644
--- a/sdks/python/apache_beam/testing/test_stream_service_test.py
+++ b/sdks/python/apache_beam/testing/test_stream_service_test.py
@@ -21,6 +21,7 @@ from __future__ import absolute_import
import sys
import unittest
+from unittest.mock import patch
import grpc
@@ -31,13 +32,6 @@ from apache_beam.portability.api.beam_interactive_api_pb2 import TestStreamFileR
from apache_beam.portability.api.beam_runner_api_pb2 import TestStreamPayload
from apache_beam.testing.test_stream_service import TestStreamServiceController
-# TODO(BEAM-8288): clean up the work-around of nose tests using Python2 without
-# unittest.mock module.
-try:
- from unittest.mock import patch
-except ImportError:
- from mock import patch # type: ignore[misc]
-
# Nose automatically detects tests if they match a regex. Here, it mistakens
# these protos as tests. For more info see the Nose docs at:
# https://nose.readthedocs.io/en/latest/writing_tests.html
diff --git a/sdks/python/apache_beam/utils/interactive_utils_test.py b/sdks/python/apache_beam/utils/interactive_utils_test.py
index 13ef93f..95a2035 100644
--- a/sdks/python/apache_beam/utils/interactive_utils_test.py
+++ b/sdks/python/apache_beam/utils/interactive_utils_test.py
@@ -22,18 +22,12 @@
from __future__ import absolute_import
import unittest
+from unittest.mock import patch
from apache_beam.runners.interactive import interactive_environment as ie
from apache_beam.runners.interactive.testing.mock_ipython import mock_get_ipython
from apache_beam.utils.interactive_utils import is_in_ipython
-# TODO(BEAM-8288): clean up the work-around of nose tests using Python2 without
-# unittest.mock module.
-try:
- from unittest.mock import patch
-except ImportError:
- from mock import patch # type: ignore[misc]
-
def unavailable_ipython():
# ModuleNotFoundError since Py3.6 is sub class of ImportError. An example,