You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2016/09/15 17:14:56 UTC
[1/2] ambari git commit: AMBARI-18366 - YAML Maps For Storm Are Not
Being Escaped Correctly (jonathanhurley)
Repository: ambari
Updated Branches:
refs/heads/branch-2.4 3f316575c -> 5cd1d306f
AMBARI-18366 - YAML Maps For Storm Are Not Being Escaped Correctly (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/19e37155
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/19e37155
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/19e37155
Branch: refs/heads/branch-2.4
Commit: 19e37155b2a435e898550b5d3e9e5d586c0bc965
Parents: 3f31657
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Mon Sep 12 15:10:15 2016 -0400
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Thu Sep 15 13:14:24 2016 -0400
----------------------------------------------------------------------
.../main/python/ambari_commons/yaml_utils.py | 45 +++++++++++++++-----
ambari-server/src/test/python/TestYAMLUtils.py | 40 ++++++++++++++++-
2 files changed, 73 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/19e37155/ambari-common/src/main/python/ambari_commons/yaml_utils.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/ambari_commons/yaml_utils.py b/ambari-common/src/main/python/ambari_commons/yaml_utils.py
index bb05c8a..9753177 100644
--- a/ambari-common/src/main/python/ambari_commons/yaml_utils.py
+++ b/ambari-common/src/main/python/ambari_commons/yaml_utils.py
@@ -19,37 +19,60 @@ limitations under the License.
"""
import re
+# [a,b,c]
+REGEX_LIST = '^\w*\[.+\]\w*$'
+
+# {a: v, b: v2, c: v3}
+REGEX_DICTIONARY = '^\w*\{.+\}\w*$'
+
+"""
+storm:
+ hosts:
+ [c6401.ambari.apache.org, c6402.ambari.apache.org]
+ groups:
+ [hadoop, foo]
+"""
+REGEX_NESTED_MAPS = '^[\w+\s*:\s*\n\s*]+\[(.*?)\]+'
+
def escape_yaml_property(value):
- unquouted = False
unquouted_values = ["null", "Null", "NULL", "true", "True", "TRUE", "false",
"False", "FALSE", "YES", "Yes", "yes", "NO", "No", "no", "ON", "On", "on",
"OFF", "Off", "off"]
+ # known list of boolean/null types
if value in unquouted_values:
- unquouted = True
-
- # if is list [a,b,c] or dictionary {a: v, b: v2, c: v3}
- if re.match('^\w*\[.+\]\w*$', value) or re.match('^\w*\{.+\}\w*$', value):
- unquouted = True
+ return value
+ # quick pythonic check for integer
try:
int(value)
- unquouted = True
+ return value
except ValueError:
pass
+ # quick pythonic check for float
try:
float(value)
- unquouted = True
+ return value
except ValueError:
pass
- if not unquouted:
- value = value.replace("'", "''")
- value = "'" + value + "'"
+ # if is list [a,b,c] or dictionary {a: v, b: v2, c: v3}
+ if re.match(REGEX_LIST, value) or re.match(REGEX_DICTIONARY, value):
+ return value
+ # check for a nested map
+ if re.match(REGEX_NESTED_MAPS, value):
+ # nested maps must begin on a newline and not have whitespace on the first line
+ value = value.lstrip()
+ return "\n" + value
+
+ # no more checks, so assume it's a string a quote it
+ value = value.replace("'", "''")
+ value = "'" + value + "'"
return value
+
def get_values_from_yaml_array(yaml_array):
"""
Converts a YAML array into a normal array of values. For example, this
http://git-wip-us.apache.org/repos/asf/ambari/blob/19e37155/ambari-server/src/test/python/TestYAMLUtils.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/TestYAMLUtils.py b/ambari-server/src/test/python/TestYAMLUtils.py
index bdbb11f..c6ee343 100644
--- a/ambari-server/src/test/python/TestYAMLUtils.py
+++ b/ambari-server/src/test/python/TestYAMLUtils.py
@@ -41,4 +41,42 @@ class TestYAMLUtils(TestCase):
self.assertEquals(expected_values, values)
values = yaml_utils.get_values_from_yaml_array('[\'c6401.ambari.apache.org\', "c6402.ambari.apache.org"]')
- self.assertEquals(expected_values, values)
\ No newline at end of file
+ self.assertEquals(expected_values, values)
+
+
+ def test_yaml_property_escaping(self):
+ """
+ Tests that YAML values are escaped with quotes properly when needed
+ """
+ self.assertEquals("True", yaml_utils.escape_yaml_property("True"))
+ self.assertEquals("FALSE", yaml_utils.escape_yaml_property("FALSE"))
+ self.assertEquals("yes", yaml_utils.escape_yaml_property("yes"))
+ self.assertEquals("NO", yaml_utils.escape_yaml_property("NO"))
+ self.assertEquals("28", yaml_utils.escape_yaml_property("28"))
+ self.assertEquals("28.0", yaml_utils.escape_yaml_property("28.0"))
+ self.assertEquals("[a,b,c]", yaml_utils.escape_yaml_property("[a,b,c]"))
+ self.assertEquals("{ foo : bar }", yaml_utils.escape_yaml_property("{ foo : bar }"))
+
+ # some strings which should be escaped
+ self.assertEquals("'5f'", yaml_utils.escape_yaml_property("5f"))
+ self.assertEquals("'28.O'", yaml_utils.escape_yaml_property("28.O"))
+ self.assertEquals("'This is a test of a string'", yaml_utils.escape_yaml_property("This is a test of a string"))
+
+ # test maps
+ map = """
+ storm:
+ hosts:
+ [c6401.ambari.apache.org, c6402.ambari.apache.org]
+ groups:
+ [hadoop, foo]
+ foo:
+ [bar, baz]
+ foo2:
+ bar2:
+ [baz2]
+ """
+ escaped_map = yaml_utils.escape_yaml_property(map)
+ self.assertTrue(escaped_map.startswith("\n"))
+ self.assertFalse("'" in escaped_map)
+
+
[2/2] ambari git commit: AMBARI-18403 - YAML Maps Can Include Dashes
and Other Non-Word Characters (jonathanhurley)
Posted by jo...@apache.org.
AMBARI-18403 - YAML Maps Can Include Dashes and Other Non-Word Characters (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/5cd1d306
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/5cd1d306
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/5cd1d306
Branch: refs/heads/branch-2.4
Commit: 5cd1d306ff95d534414707f10faedc6e1b1437f6
Parents: 19e3715
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Thu Sep 15 11:51:12 2016 -0400
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Thu Sep 15 13:14:40 2016 -0400
----------------------------------------------------------------------
.../main/python/ambari_commons/yaml_utils.py | 16 ++++++++---
ambari-server/src/test/python/TestYAMLUtils.py | 30 ++++++++++++++------
2 files changed, 34 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/5cd1d306/ambari-common/src/main/python/ambari_commons/yaml_utils.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/ambari_commons/yaml_utils.py b/ambari-common/src/main/python/ambari_commons/yaml_utils.py
index 9753177..dae5b56 100644
--- a/ambari-common/src/main/python/ambari_commons/yaml_utils.py
+++ b/ambari-common/src/main/python/ambari_commons/yaml_utils.py
@@ -26,13 +26,21 @@ REGEX_LIST = '^\w*\[.+\]\w*$'
REGEX_DICTIONARY = '^\w*\{.+\}\w*$'
"""
-storm:
+storm-cluster:
hosts:
- [c6401.ambari.apache.org, c6402.ambari.apache.org]
+ [c6401.ambari.apache.org, c6402.ambari.apache.org, c6403-master.ambari.apache.org]
groups:
- [hadoop, foo]
+ [hadoop, hadoop-secure]
+
+^\s* - allow any whitespace or newlines to start
+\S+ - at least 1 word character (including dashes)
+[ ]*:[ ]* - followed by a colon (allowing spaces around the colon)
+[\r\n\f]+ - at least 1 newline
+
+\s*\S+[ ]*:[ ]*[\r\n\f] - follow with the same basically to ensure a map of maps
"""
-REGEX_NESTED_MAPS = '^[\w+\s*:\s*\n\s*]+\[(.*?)\]+'
+REGEX_NESTED_MAPS = "^\s*\S+[ ]*:[ ]*[\r\n\f]+\s*\S+[ ]*:[ ]*[\r\n\f]"
+
def escape_yaml_property(value):
unquouted_values = ["null", "Null", "NULL", "true", "True", "TRUE", "false",
http://git-wip-us.apache.org/repos/asf/ambari/blob/5cd1d306/ambari-server/src/test/python/TestYAMLUtils.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/TestYAMLUtils.py b/ambari-server/src/test/python/TestYAMLUtils.py
index c6ee343..4d2d035 100644
--- a/ambari-server/src/test/python/TestYAMLUtils.py
+++ b/ambari-server/src/test/python/TestYAMLUtils.py
@@ -64,19 +64,33 @@ class TestYAMLUtils(TestCase):
# test maps
map = """
- storm:
+ storm-cluster:
hosts:
- [c6401.ambari.apache.org, c6402.ambari.apache.org]
+ [c6401.ambari.apache.org, c6402.ambari.apache.org, c6403-master.ambari.apache.org]
groups:
- [hadoop, foo]
- foo:
- [bar, baz]
- foo2:
- bar2:
- [baz2]
+ [hadoop, hadoop-secure]
"""
escaped_map = yaml_utils.escape_yaml_property(map)
self.assertTrue(escaped_map.startswith("\n"))
self.assertFalse("'" in escaped_map)
+ # try some weird but valid formatting
+ map = """
+
+
+ storm-cluster :
+ hosts :
+[c6401.ambari.apache.org, c6402.ambari.apache.org, c6403-master.ambari.apache.org]
+ groups :
+ [hadoop!!!, hadoop-secure!!!!-----]
+ """
+ escaped_map = yaml_utils.escape_yaml_property(map)
+ self.assertTrue(escaped_map.startswith("\n"))
+ self.assertFalse("'" in escaped_map)
+ # try some bad formatting - this is not a map
+ map = """ foo : bar :
+ [baz]"""
+ escaped_map = yaml_utils.escape_yaml_property(map)
+ self.assertFalse(escaped_map.startswith("\n"))
+ self.assertTrue("'" in escaped_map)