You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ro...@apache.org on 2016/07/23 23:43:06 UTC
[2/4] incubator-beam git commit: Make save_main_session optional
Make save_main_session optional
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/2e36602c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/2e36602c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/2e36602c
Branch: refs/heads/python-sdk
Commit: 2e36602c0a2944129698866cc31abc3bcac6169f
Parents: af6d380
Author: Silviu Calinoiu <si...@google.com>
Authored: Sat Jul 23 09:31:30 2016 -0700
Committer: Silviu Calinoiu <si...@google.com>
Committed: Sat Jul 23 09:31:30 2016 -0700
----------------------------------------------------------------------
.../python/apache_beam/utils/dependency_test.py | 33 ++++++++++++--------
sdks/python/apache_beam/utils/options.py | 2 +-
2 files changed, 21 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2e36602c/sdks/python/apache_beam/utils/dependency_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/utils/dependency_test.py b/sdks/python/apache_beam/utils/dependency_test.py
index b7a9296..2f9a57b 100644
--- a/sdks/python/apache_beam/utils/dependency_test.py
+++ b/sdks/python/apache_beam/utils/dependency_test.py
@@ -81,10 +81,12 @@ class SetupTest(unittest.TestCase):
[],
dependency.stage_job_resources(options))
- def test_default_resources(self):
+ def test_with_main_session(self):
staging_dir = tempfile.mkdtemp()
options = PipelineOptions()
+
options.view_as(GoogleCloudOptions).staging_location = staging_dir
+ options.view_as(SetupOptions).save_main_session = True
self.update_options(options)
self.assertEqual(
@@ -94,6 +96,16 @@ class SetupTest(unittest.TestCase):
os.path.isfile(
os.path.join(staging_dir, names.PICKLED_MAIN_SESSION_FILE)))
+ def test_default_resources(self):
+ staging_dir = tempfile.mkdtemp()
+ options = PipelineOptions()
+ options.view_as(GoogleCloudOptions).staging_location = staging_dir
+ self.update_options(options)
+
+ self.assertEqual(
+ [],
+ dependency.stage_job_resources(options))
+
def test_with_requirements_file(self):
staging_dir = tempfile.mkdtemp()
source_dir = tempfile.mkdtemp()
@@ -106,7 +118,7 @@ class SetupTest(unittest.TestCase):
self.create_temp_file(
os.path.join(source_dir, dependency.REQUIREMENTS_FILE), 'nothing')
self.assertEqual(
- sorted([dependency.REQUIREMENTS_FILE, names.PICKLED_MAIN_SESSION_FILE,
+ sorted([dependency.REQUIREMENTS_FILE,
'abc.txt', 'def.txt']),
sorted(dependency.stage_job_resources(
options,
@@ -145,7 +157,7 @@ class SetupTest(unittest.TestCase):
self.create_temp_file(
os.path.join(source_dir, dependency.REQUIREMENTS_FILE), 'nothing')
self.assertEqual(
- sorted([dependency.REQUIREMENTS_FILE, names.PICKLED_MAIN_SESSION_FILE,
+ sorted([dependency.REQUIREMENTS_FILE,
'abc.txt', 'def.txt']),
sorted(dependency.stage_job_resources(
options,
@@ -169,8 +181,7 @@ class SetupTest(unittest.TestCase):
source_dir, 'setup.py')
self.assertEqual(
- [dependency.WORKFLOW_TARBALL_FILE,
- names.PICKLED_MAIN_SESSION_FILE],
+ [dependency.WORKFLOW_TARBALL_FILE],
dependency.stage_job_resources(
options,
# We replace the build setup command because a realistic one would
@@ -265,8 +276,7 @@ class SetupTest(unittest.TestCase):
options.view_as(SetupOptions).sdk_location = 'default'
self.assertEqual(
- [names.PICKLED_MAIN_SESSION_FILE,
- names.DATAFLOW_SDK_TARBALL_FILE],
+ [names.DATAFLOW_SDK_TARBALL_FILE],
dependency.stage_job_resources(
options,
file_copy=dependency._dependency_file_copy))
@@ -286,8 +296,7 @@ class SetupTest(unittest.TestCase):
options.view_as(SetupOptions).sdk_location = sdk_location
self.assertEqual(
- [names.PICKLED_MAIN_SESSION_FILE,
- names.DATAFLOW_SDK_TARBALL_FILE],
+ [names.DATAFLOW_SDK_TARBALL_FILE],
dependency.stage_job_resources(options))
tarball_path = os.path.join(
staging_dir, names.DATAFLOW_SDK_TARBALL_FILE)
@@ -321,8 +330,7 @@ class SetupTest(unittest.TestCase):
options.view_as(SetupOptions).sdk_location = sdk_location
self.assertEqual(
- [names.PICKLED_MAIN_SESSION_FILE,
- names.DATAFLOW_SDK_TARBALL_FILE],
+ [names.DATAFLOW_SDK_TARBALL_FILE],
dependency.stage_job_resources(options))
def test_with_extra_packages(self):
@@ -363,8 +371,7 @@ class SetupTest(unittest.TestCase):
self.assertEqual(
['abc.tar.gz', 'xyz.tar.gz', 'xyz2.tar', 'gcs.tar.gz',
- dependency.EXTRA_PACKAGES_FILE,
- names.PICKLED_MAIN_SESSION_FILE],
+ dependency.EXTRA_PACKAGES_FILE],
dependency.stage_job_resources(options))
with open(os.path.join(staging_dir, dependency.EXTRA_PACKAGES_FILE)) as f:
self.assertEqual(['abc.tar.gz\n', 'xyz.tar.gz\n', 'xyz2.tar\n',
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2e36602c/sdks/python/apache_beam/utils/options.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/utils/options.py b/sdks/python/apache_beam/utils/options.py
index 335beea..933aa12 100644
--- a/sdks/python/apache_beam/utils/options.py
+++ b/sdks/python/apache_beam/utils/options.py
@@ -412,7 +412,7 @@ class SetupOptions(PipelineOptions):
'install the resulting package before running any custom code.'))
parser.add_argument(
'--save_main_session',
- default=True,
+ default=False,
action='store_true',
help=
('Save the main session state so that pickled functions and classes '