You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by al...@apache.org on 2019/06/19 21:21:43 UTC
[beam] branch master updated: Add common code infrastructure for
element-wise snippets
This is an automated email from the ASF dual-hosted git repository.
altay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new c32acce Add common code infrastructure for element-wise snippets
new 3124531 Merge pull request #8811 from davidcavazos/element-wise-snippets
c32acce is described below
commit c32accec7b6d254851d156facf03154198cf2e54
Author: David Cavazos <da...@gmail.com>
AuthorDate: Mon Jun 10 16:01:56 2019 -0700
Add common code infrastructure for element-wise snippets
---
.../examples/snippets/transforms/__init__.py | 18 ++++++
.../snippets/transforms/element_wise/__init__.py | 18 ++++++
sdks/python/apache_beam/examples/snippets/util.py | 56 ++++++++++++++++++
.../apache_beam/examples/snippets/util_test.py | 67 ++++++++++++++++++++++
4 files changed, 159 insertions(+)
diff --git a/sdks/python/apache_beam/examples/snippets/transforms/__init__.py b/sdks/python/apache_beam/examples/snippets/transforms/__init__.py
new file mode 100644
index 0000000..6569e3f
--- /dev/null
+++ b/sdks/python/apache_beam/examples/snippets/transforms/__init__.py
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from __future__ import absolute_import
diff --git a/sdks/python/apache_beam/examples/snippets/transforms/element_wise/__init__.py b/sdks/python/apache_beam/examples/snippets/transforms/element_wise/__init__.py
new file mode 100644
index 0000000..6569e3f
--- /dev/null
+++ b/sdks/python/apache_beam/examples/snippets/transforms/element_wise/__init__.py
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from __future__ import absolute_import
diff --git a/sdks/python/apache_beam/examples/snippets/util.py b/sdks/python/apache_beam/examples/snippets/util.py
new file mode 100644
index 0000000..6e6e9e0
--- /dev/null
+++ b/sdks/python/apache_beam/examples/snippets/util.py
@@ -0,0 +1,56 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from __future__ import absolute_import
+
+import argparse
+import shlex
+import subprocess as sp
+
+
+def parse_example(argv=None):
+ """Parse the command line arguments and return it as a string function call.
+
+ Examples:
+ python path/to/snippets.py function_name
+ python path/to/snippets.py function_name arg1
+ python path/to/snippets.py function_name arg1 arg2 ... argN
+ """
+ parser = argparse.ArgumentParser()
+ parser.add_argument('example', help='Name of the example to run.')
+ parser.add_argument('args', nargs=argparse.REMAINDER,
+ help='Arguments for example.')
+ args = parser.parse_args(argv)
+
+ # Return the example as a string representing the Python function call.
+ example_args = ', '.join([repr(arg) for arg in args.args])
+ return '{}({})'.format(args.example, example_args)
+
+
+def run_shell_commands(commands, **kwargs):
+ """Runs a list of Notebook-like shell commands.
+
+ Lines starting with `#` are ignored as comments.
+ Lines starting with `!` are run as commands.
+ Variables like `{variable}` are substituted with **kwargs.
+ """
+ for cmd in commands:
+ cmd = cmd.strip().lstrip('!').format(**kwargs)
+ sp_cmd = shlex.split(cmd, comments=True, posix=True)
+ if sp_cmd:
+ sp.call(sp_cmd)
+ yield sp_cmd
diff --git a/sdks/python/apache_beam/examples/snippets/util_test.py b/sdks/python/apache_beam/examples/snippets/util_test.py
new file mode 100644
index 0000000..a23e916
--- /dev/null
+++ b/sdks/python/apache_beam/examples/snippets/util_test.py
@@ -0,0 +1,67 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from __future__ import absolute_import
+
+import unittest
+
+from mock import patch
+
+from apache_beam.examples.snippets.util import *
+
+
+class UtilTest(unittest.TestCase):
+ def test_parse_example_empty(self):
+ # python path/to/snippets.py
+ argv = []
+ with self.assertRaises(SystemExit):
+ self.assertEqual(parse_example(argv), 'example()')
+
+ def test_parse_example_no_arguments(self):
+ # python path/to/snippets.py example
+ argv = ['example']
+ self.assertEqual(parse_example(argv), 'example()')
+
+ def test_parse_example_one_argument(self):
+ # python path/to/snippets.py example A
+ argv = ['example', 'A']
+ self.assertEqual(parse_example(argv), "example('A')")
+
+ def test_parse_example_multiple_arguments(self):
+ # python path/to/snippets.py example A B "C's"
+ argv = ['example', 'A', 'B', "C's"]
+ self.assertEqual(parse_example(argv), "example('A', 'B', \"C's\")")
+
+ @patch('subprocess.call', lambda cmd: None)
+ def test_run_shell_commands(self):
+ commands = [
+ ' # this is a comment ',
+ ' ! echo this is a shell command ',
+ ' !echo {variable} ',
+ ' echo "quoted arguments work" # trailing comment ',
+ ]
+ actual = list(run_shell_commands(commands, variable='hello world'))
+ expected = [
+ ['echo', 'this', 'is', 'a', 'shell', 'command'],
+ ['echo', 'hello', 'world'],
+ ['echo', 'quoted arguments work'],
+ ]
+ self.assertEqual(actual, expected)
+
+
+if __name__ == '__main__':
+ unittest.main()