You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@brooklyn.apache.org by he...@apache.org on 2015/04/21 22:41:23 UTC
[04/16] incubator-brooklyn git commit: add yaml support for
extracting yaml items with comments and replacing extracts
add yaml support for extracting yaml items with comments and replacing extracts
Project: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/commit/456049b3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/tree/456049b3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/diff/456049b3
Branch: refs/heads/master
Commit: 456049b32a83030f3a55eede27e18fa16bf7cb43
Parents: 4338a8f
Author: Alex Heneveld <al...@cloudsoftcorp.com>
Authored: Sun Mar 29 17:57:40 2015 -0500
Committer: Alex Heneveld <al...@cloudsoftcorp.com>
Committed: Wed Apr 15 20:05:19 2015 -0500
----------------------------------------------------------------------
.../src/main/java/brooklyn/util/yaml/Yamls.java | 342 ++++++++++++++++++-
.../test/java/brooklyn/util/yaml/YamlsTest.java | 127 +++++++
2 files changed, 461 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/456049b3/utils/common/src/main/java/brooklyn/util/yaml/Yamls.java
----------------------------------------------------------------------
diff --git a/utils/common/src/main/java/brooklyn/util/yaml/Yamls.java b/utils/common/src/main/java/brooklyn/util/yaml/Yamls.java
index ad56bfe..2bcbe87 100644
--- a/utils/common/src/main/java/brooklyn/util/yaml/Yamls.java
+++ b/utils/common/src/main/java/brooklyn/util/yaml/Yamls.java
@@ -19,6 +19,7 @@
package brooklyn.util.yaml;
import java.io.Reader;
+import java.io.StringReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
@@ -26,10 +27,23 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import javax.annotation.Nullable;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.yaml.snakeyaml.Yaml;
+import org.yaml.snakeyaml.error.Mark;
+import org.yaml.snakeyaml.nodes.MappingNode;
+import org.yaml.snakeyaml.nodes.Node;
+import org.yaml.snakeyaml.nodes.NodeId;
+import org.yaml.snakeyaml.nodes.NodeTuple;
+import org.yaml.snakeyaml.nodes.ScalarNode;
+import org.yaml.snakeyaml.nodes.SequenceNode;
import brooklyn.util.collections.Jsonya;
+import brooklyn.util.collections.MutableList;
+import brooklyn.util.exceptions.Exceptions;
+import brooklyn.util.text.Strings;
import com.google.common.annotations.Beta;
import com.google.common.collect.Iterables;
@@ -38,12 +52,13 @@ public class Yamls {
private static final Logger log = LoggerFactory.getLogger(Yamls.class);
- /** returns the given yaml object (map or list or primitive) as the given yaml-supperted type
- * (map or list or primitive e.g. string, number, boolean).
+ /** returns the given (yaml-parsed) object as the given yaml type.
+ * <p>
+ * if the object is an iterable or iterator this method will fully expand it as a list.
+ * if the requested type is not an iterable or iterator, and the list contains a single item, this will take that single item.
+ * <p>
+ * in other cases this method simply does a type-check and cast (no other type coercion).
* <p>
- * if the object is an iterable containing a single element, and the type is not an iterable,
- * this will attempt to unwrap it.
- *
* @throws IllegalArgumentException if the input is an iterable not containing a single element,
* and the cast is requested to a non-iterable type
* @throws ClassCastException if cannot be casted */
@@ -61,12 +76,12 @@ public class Yamls {
while (xi.hasNext()) {
result.add( xi.next() );
}
- if (type.isAssignableFrom(Iterable.class)) return (T)result;
- if (type.isAssignableFrom(Iterator.class)) return (T)result.iterator();
if (type.isAssignableFrom(List.class)) return (T)result;
+ if (type.isAssignableFrom(Iterator.class)) return (T)result.iterator();
x = Iterables.getOnlyElement(result);
}
- return (T)x;
+ if (type.isInstance(x)) return (T)x;
+ throw new ClassCastException("Cannot convert "+x+" to "+type);
}
/**
@@ -175,4 +190,315 @@ public class Yamls {
}
return result;
}
+
+ @Beta
+ public static class YamlExtract {
+ String yaml;
+ NodeTuple focusTuple;
+ Node prev, key, focus, next;
+ Exception error;
+ boolean includeKey = false, includePrecedingComments = true, includeOriginalIndentation = false;
+
+ private int indexStart(Node node, boolean defaultIsYamlEnd) {
+ if (node==null) return defaultIsYamlEnd ? yaml.length() : 0;
+ return index(node.getStartMark());
+ }
+ private int indexEnd(Node node, boolean defaultIsYamlEnd) {
+ if (!found() || node==null) return defaultIsYamlEnd ? yaml.length() : 0;
+ return index(node.getEndMark());
+ }
+ private int index(Mark mark) {
+ try {
+ return mark.getIndex();
+ } catch (NoSuchMethodError e) {
+ throw new IllegalStateException("Class version error. This can happen if using a TestNG plugin in your IDE "
+ + "which is an older version, dragging in an older version of SnakeYAML which does not support Mark.getIndex.", e);
+ }
+ }
+
+ public int getEndOfPrevious() {
+ return indexEnd(prev, false);
+ }
+ @Nullable public Node getKey() {
+ return key;
+ }
+ public Node getResult() {
+ return focus;
+ }
+ public int getStartOfThis() {
+ if (includeKey && focusTuple!=null) return indexStart(focusTuple.getKeyNode(), false);
+ return indexStart(focus, false);
+ }
+ private int getStartColumnOfThis() {
+ if (includeKey && focusTuple!=null) return focusTuple.getKeyNode().getStartMark().getColumn();
+ return focus.getStartMark().getColumn();
+ }
+ public int getEndOfThis() {
+ return getEndOfThis(false);
+ }
+ private int getEndOfThis(boolean goToEndIfNoNext) {
+ if (next==null && goToEndIfNoNext) return yaml.length();
+ return indexEnd(focus, false);
+ }
+ public int getStartOfNext() {
+ return indexStart(next, true);
+ }
+
+ private static int initialWhitespaceLength(String x) {
+ int i=0;
+ while (i < x.length() && x.charAt(i)==' ') i++;
+ return i;
+ }
+
+ public String getFullYamlTextOriginal() {
+ return yaml;
+ }
+
+ /** Returns the original YAML with the found item replaced by the given replacement YAML.
+ * @param replacement YAML to put in for the found item;
+ * this YAML typically should not have any special indentation -- if required when replacing it will be inserted.
+ * <p>
+ * if replacing an inline map entry, the supplied entry must follow the structure being replaced;
+ * for example, if replacing the value in <code>key: value</code> with a map,
+ * supplying a replacement <code>subkey: value</code> would result in invalid yaml;
+ * the replacement must be supplied with a newline, either before the subkey or after.
+ * (if unsure we believe it is always valid to include an initial newline or comment with newline.)
+ */
+ public String getFullYamlTextWithExtractReplaced(String replacement) {
+ if (!found()) throw new IllegalStateException("Cannot perform replacement when item was not matched.");
+ String result = yaml.substring(0, getStartOfThis());
+
+ String[] newLines = replacement.split("\n");
+ for (int i=1; i<newLines.length; i++)
+ newLines[i] = Strings.makePaddedString("", getStartColumnOfThis(), "", " ") + newLines[i];
+ result += Strings.lines(newLines);
+ if (replacement.endsWith("\n")) result += "\n";
+
+ int end = getEndOfThis();
+ result += yaml.substring(end);
+
+ return result;
+ }
+
+ /** Specifies whether the key should be included in the found text,
+ * when calling {@link #getMatchedYamlText()} or {@link #getFullYamlTextWithExtractReplaced(String)},
+ * if the found item is a map entry.
+ * Defaults to false.
+ * @return this object, for use in a fluent constructions
+ */
+ public YamlExtract withKeyIncluded(boolean includeKey) {
+ this.includeKey = includeKey;
+ return this;
+ }
+
+ /** Specifies whether comments preceding the found item should be included,
+ * when calling {@link #getMatchedYamlText()} or {@link #getFullYamlTextWithExtractReplaced(String)}.
+ * This will not include comments which are indented further than the item,
+ * as those will typically be aligned with the previous item
+ * (whereas comments whose indentation is the same or less than the found item
+ * will typically be aligned with this item).
+ * Defaults to true.
+ * @return this object, for use in a fluent constructions
+ */
+ public YamlExtract withPrecedingCommentsIncluded(boolean includePrecedingComments) {
+ this.includePrecedingComments = includePrecedingComments;
+ return this;
+ }
+
+ /** Specifies whether the original indentation should be preserved
+ * (and in the case of the first line, whether whitespace should be inserted so its start column is preserved),
+ * when calling {@link #getMatchedYamlText()}.
+ * Defaults to false, the returned text will be outdented as far as possible.
+ * @return this object, for use in a fluent constructions
+ */
+ public YamlExtract withOriginalIndentation(boolean includeOriginalIndentation) {
+ this.includeOriginalIndentation = includeOriginalIndentation;
+ return this;
+ }
+
+ @Beta
+ public String getMatchedYamlText() {
+ if (!found()) return null;
+
+ String[] body = yaml.substring(getStartOfThis(), getEndOfThis(true)).split("\n", -1);
+
+ int firstLineIndentationOfFirstThing;
+ if (focusTuple!=null) {
+ firstLineIndentationOfFirstThing = focusTuple.getKeyNode().getStartMark().getColumn();
+ } else {
+ firstLineIndentationOfFirstThing = focus.getStartMark().getColumn();
+ }
+ int firstLineIndentationToAdd;
+ if (focusTuple!=null && (includeKey || body.length==1)) {
+ firstLineIndentationToAdd = focusTuple.getKeyNode().getStartMark().getColumn();
+ } else {
+ firstLineIndentationToAdd = focus.getStartMark().getColumn();
+ }
+
+
+ String firstLineIndentationToAddS = Strings.makePaddedString("", firstLineIndentationToAdd, "", " ");
+ String subsequentLineIndentationToRemoveS = firstLineIndentationToAddS;
+
+/* complexities of indentation:
+
+x: a
+ bc
+
+should become
+
+a
+ bc
+
+whereas
+
+- a: 0
+ b: 1
+
+selecting 0 should give
+
+a: 0
+b: 1
+
+ */
+ List<String> result = MutableList.of();
+ if (includePrecedingComments) {
+ String[] preceding = yaml.substring(getEndOfPrevious(), getStartOfThis()).split("\n");
+ // suppress comments which are on the same line as the previous item or indented more than firstLineIndentation,
+ // ensuring appropriate whitespace is added to preceding[0] if it starts mid-line
+ if (preceding.length>0 && prev!=null) {
+ preceding[0] = Strings.makePaddedString("", prev.getEndMark().getColumn(), "", " ") + preceding[0];
+ }
+ for (String p: preceding) {
+ int w = initialWhitespaceLength(p);
+ p = p.substring(w);
+ if (p.startsWith("#")) {
+ // only add if the hash is not indented further than the first line
+ if (w <= firstLineIndentationOfFirstThing) {
+ if (includeOriginalIndentation) p = firstLineIndentationToAddS + p;
+ result.add(p);
+ }
+ }
+ }
+ }
+
+ boolean doneFirst = false;
+ for (String p: body) {
+ if (!doneFirst) {
+ if (includeOriginalIndentation) {
+ // have to insert the right amount of spacing
+ p = firstLineIndentationToAddS + p;
+ }
+ result.add(p);
+ doneFirst = true;
+ } else {
+ if (includeOriginalIndentation) {
+ result.add(p);
+ } else {
+ result.add(Strings.removeFromStart(p, subsequentLineIndentationToRemoveS));
+ }
+ }
+ }
+ return Strings.join(result, "\n");
+ }
+
+ boolean found() {
+ return focus != null;
+ }
+
+ public Exception getError() {
+ return error;
+ }
+
+ @Override
+ public String toString() {
+ return "Extract["+focus+";prev="+prev+";key="+key+";next="+next+"]";
+ }
+ }
+
+ private static void findTextOfYamlAtPath(YamlExtract result, int pathIndex, Object ...path) {
+ if (pathIndex>=path.length) {
+ // we're done
+ return;
+ }
+
+ Object pathItem = path[pathIndex];
+ Node node = result.focus;
+
+ if (node.getNodeId()==NodeId.mapping && pathItem instanceof String) {
+ // find key
+ Iterator<NodeTuple> ti = ((MappingNode)node).getValue().iterator();
+ while (ti.hasNext()) {
+ NodeTuple t = ti.next();
+ Node key = t.getKeyNode();
+ if (key.getNodeId()==NodeId.scalar) {
+ if (pathItem.equals( ((ScalarNode)key).getValue() )) {
+ result.key = key;
+ result.focus = t.getValueNode();
+ if (pathIndex+1<path.length) {
+ // there are more path items, so the key here is a previous node
+ result.prev = key;
+ } else {
+ result.focusTuple = t;
+ }
+ findTextOfYamlAtPath(result, pathIndex+1, path);
+ if (result.next==null) {
+ if (ti.hasNext()) result.next = ti.next().getKeyNode();
+ }
+ return;
+ } else {
+ result.prev = t.getValueNode();
+ }
+ } else {
+ throw new IllegalStateException("Key "+key+" is not a scalar, searching for "+pathItem+" at depth "+pathIndex+" of "+Arrays.asList(path));
+ }
+ }
+ throw new IllegalStateException("Did not find "+pathItem+" in "+node+" at depth "+pathIndex+" of "+Arrays.asList(path));
+
+ } else if (node.getNodeId()==NodeId.sequence && pathItem instanceof Number) {
+ // find list item
+ List<Node> nl = ((SequenceNode)node).getValue();
+ int i = ((Number)pathItem).intValue();
+ if (i>=nl.size())
+ throw new IllegalStateException("Index "+i+" is out of bounds in "+node+", searching for "+pathItem+" at depth "+pathIndex+" of "+Arrays.asList(path));
+ if (i>0) result.prev = nl.get(i-1);
+ result.key = null;
+ result.focus = nl.get(i);
+ findTextOfYamlAtPath(result, pathIndex+1, path);
+ if (result.next==null) {
+ if (nl.size()>i+1) result.next = nl.get(i+1);
+ }
+ return;
+
+ } else {
+ throw new IllegalStateException("Node "+node+" does not match selector "+pathItem+" at depth "+pathIndex+" of "+Arrays.asList(path));
+ }
+
+ // unreachable
+ }
+
+
+ /** Given a path, where each segment consists of a string (key) or number (element in list),
+ * this will find the YAML text for that element
+ * <p>
+ * If not found this will return a {@link YamlExtract}
+ * where {@link YamlExtract#isMatch()} is false and {@link YamlExtract#getError()} is set. */
+ public static YamlExtract getTextOfYamlAtPath(String yaml, Object ...path) {
+ YamlExtract result = new YamlExtract();
+ try {
+ int pathIndex = 0;
+ result.yaml = yaml;
+ result.focus = new Yaml().compose(new StringReader(yaml));
+
+ findTextOfYamlAtPath(result, pathIndex, path);
+ return result;
+ } catch (NoSuchMethodError e) {
+ throw new IllegalStateException("Class version error. This can happen if using a TestNG plugin in your IDE "
+ + "which is an older version, dragging in an older version of SnakeYAML which does not support Mark.getIndex.", e);
+ } catch (Exception e) {
+ Exceptions.propagateIfFatal(e);
+ log.debug("Unable to find element in yaml (setting in result): "+e);
+ result.error = e;
+ return result;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/456049b3/utils/common/src/test/java/brooklyn/util/yaml/YamlsTest.java
----------------------------------------------------------------------
diff --git a/utils/common/src/test/java/brooklyn/util/yaml/YamlsTest.java b/utils/common/src/test/java/brooklyn/util/yaml/YamlsTest.java
index 27437d9..36c146b 100644
--- a/utils/common/src/test/java/brooklyn/util/yaml/YamlsTest.java
+++ b/utils/common/src/test/java/brooklyn/util/yaml/YamlsTest.java
@@ -20,14 +20,30 @@ package brooklyn.util.yaml;
import static org.testng.Assert.assertEquals;
+import java.util.Iterator;
+import java.util.List;
+
+import org.testng.Assert;
+import org.testng.TestNG;
import org.testng.annotations.Test;
+import brooklyn.util.collections.MutableList;
+
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
public class YamlsTest {
@Test
+ public void testGetAs() throws Exception {
+ MutableList<String> list = MutableList.of("x");
+ assertEquals(Yamls.getAs(list.iterator(), List.class), list);
+ assertEquals(Yamls.getAs(list.iterator(), Iterable.class), list);
+ assertEquals(Yamls.getAs(list.iterator(), Iterator.class), list.iterator());
+ assertEquals(Yamls.getAs(list.iterator(), String.class), "x");
+ }
+
+ @Test
public void testGetAt() throws Exception {
// leaf of map
assertEquals(Yamls.getAt("k1: v", ImmutableList.of("k1")), "v");
@@ -44,4 +60,115 @@ public class YamlsTest {
assertEquals(Yamls.getAt("k1: [v1, v2]", ImmutableList.<String>of("k1", "[0]")), "v1");
assertEquals(Yamls.getAt("k1: [v1, v2]", ImmutableList.<String>of("k1", "[1]")), "v2");
}
+
+
+ @Test
+ public void testExtractMap() {
+ String sample = "#before\nk1:\n- v1\nk2:\n # comment\n k21: v21\nk3: v3\n#after\n";
+
+ Assert.assertEquals(Yamls.getTextOfYamlAtPath(sample, "k1").withKeyIncluded(true).getMatchedYamlText(),
+ sample.substring(0, sample.indexOf("k2")));
+ Assert.assertEquals(Yamls.getTextOfYamlAtPath(sample, "k3").withKeyIncluded(true).getMatchedYamlText(),
+ sample.substring(sample.indexOf("k3")));
+
+ // comments and no key, outdented - the default
+ Assert.assertEquals(Yamls.getTextOfYamlAtPath(sample, "k2", "k21").getMatchedYamlText(),
+ "# comment\nv21");
+ Assert.assertEquals(Yamls.getTextOfYamlAtPath(sample, "k2", "k21").getMatchedYamlText(),
+ "# comment\nv21");
+ // comments and key
+ Assert.assertEquals(Yamls.getTextOfYamlAtPath(sample, "k2", "k21").withKeyIncluded(true).getMatchedYamlText(),
+ "# comment\nk21: v21");
+ // no comments
+ Assert.assertEquals(Yamls.getTextOfYamlAtPath(sample, "k2", "k21").withKeyIncluded(true).withPrecedingCommentsIncluded(false).getMatchedYamlText(),
+ "k21: v21");
+ // no comments and no key
+ Assert.assertEquals(Yamls.getTextOfYamlAtPath(sample, "k2", "k21").withPrecedingCommentsIncluded(false).getMatchedYamlText(),
+ "v21");
+
+ // comments and no key, not outdented
+ Assert.assertEquals(Yamls.getTextOfYamlAtPath(sample, "k2", "k21").withOriginalIndentation(true).getMatchedYamlText(),
+ " # comment\n v21");
+ // comments and key
+ Assert.assertEquals(Yamls.getTextOfYamlAtPath(sample, "k2", "k21").withKeyIncluded(true).withOriginalIndentation(true).getMatchedYamlText(),
+ " # comment\n k21: v21");
+ // no comments
+ Assert.assertEquals(Yamls.getTextOfYamlAtPath(sample, "k2", "k21").withKeyIncluded(true).withPrecedingCommentsIncluded(false).withOriginalIndentation(true).getMatchedYamlText(),
+ " k21: v21");
+ // no comments and no key
+ Assert.assertEquals(Yamls.getTextOfYamlAtPath(sample, "k2", "k21").withPrecedingCommentsIncluded(false).withOriginalIndentation(true).getMatchedYamlText(),
+ " v21");
+ }
+
+ @Test
+ public void testExtractInList() {
+ String sample =
+ "- a\n" +
+ "- b: 2\n" +
+ "- # c\n" +
+ " c1:\n" +
+ " 1\n" +
+ " c2:\n" +
+ " 2\n" +
+ "-\n" +
+ " - a # for a\n" +
+ " # for b\n" +
+ " - b\n";
+
+ Assert.assertEquals(Yamls.getTextOfYamlAtPath(sample, 0).getMatchedYamlText(), "a");
+ Assert.assertEquals(Yamls.getTextOfYamlAtPath(sample, 1, "b").getMatchedYamlText(), "2");
+ Assert.assertEquals(Yamls.getTextOfYamlAtPath(sample, 3, 0).getMatchedYamlText(),
+ "a"
+ // TODO comments after on same line not yet included - would be nice to add
+// "a # for a"
+ );
+
+ // out-dent
+ Assert.assertEquals(Yamls.getTextOfYamlAtPath(sample, 2).getMatchedYamlText(), "c1:\n 1\nc2:\n 2\n");
+ // don't outdent
+ Assert.assertEquals(Yamls.getTextOfYamlAtPath(sample, 2).withOriginalIndentation(true).getMatchedYamlText(), " c1:\n 1\n c2:\n 2\n");
+ Assert.assertEquals(Yamls.getTextOfYamlAtPath(sample, 3, 0).withOriginalIndentation(true).getMatchedYamlText(),
+ " a"
+ // as above, comments after not included
+// " a # for a"
+ );
+
+ // with preceding comments
+ // TODO final item includes newline (and comments) after - this behaviour might change, it's inconsistent,
+ // but it means the final comments aren't lost
+ Assert.assertEquals(Yamls.getTextOfYamlAtPath(sample, 3, 1).getMatchedYamlText(), "# for b\nb\n");
+
+ // exclude preceding comments
+ Assert.assertEquals(Yamls.getTextOfYamlAtPath(sample, 3, 1).withPrecedingCommentsIncluded(false).getMatchedYamlText(), "b\n");
+ }
+
+ @Test
+ public void testExtractMapIgnoringPreviousComments() {
+ String sample = "a: 1 # one\n"
+ + "b: 2 # two";
+ Assert.assertEquals(Yamls.getTextOfYamlAtPath(sample, "b").getMatchedYamlText(),
+ "2 # two");
+ }
+
+ @Test
+ public void testExtractMapWithOddWhitespace() {
+ Assert.assertEquals(Yamls.getTextOfYamlAtPath("x: a\n bc", "x").getMatchedYamlText(),
+ "a\n bc");
+ }
+
+ @Test
+ public void testReplace() {
+ Assert.assertEquals(Yamls.getTextOfYamlAtPath("x: a\n bc", "x").getFullYamlTextWithExtractReplaced("\nc: 1\nd: 2"),
+ "x: \n c: 1\n d: 2");
+ }
+
+ // convenience, since running with older TestNG IDE plugin will fail (older snakeyaml dependency);
+ // if you run as a java app it doesn't bring in the IDE TestNG jar version, and it works
+ public static void main(String[] args) {
+ TestNG testng = new TestNG();
+ testng.setTestClasses(new Class[] { YamlsTest.class });
+// testng.setVerbose(9);
+ testng.run();
+ }
+
}