You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2018/11/23 19:42:10 UTC
[3/4] impala git commit: IMPALA-7881: Visualize AST for easier
debugging
IMPALA-7881: Visualize AST for easier debugging
Provides a debug-time class to visualize the analysis-time AST for use
by developers. Uses a very simple JSON-like output. See IMPALA-7881 for
an example. Emphasize simplicity over elegance, many refinements can
follow.
Tests: This is test only code that is not even called from anywhere. A
developer must insert a temporary call when doing debugging. As a
result, the code cannot impact any other part of the code base.
Change-Id: Ia083c63953ceec0abedd2e0753b9444c19fcd114
Reviewed-on: http://gerrit.cloudera.org:8080/11980
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-by: Lars Volker <lv...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/aa294cf6
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/aa294cf6
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/aa294cf6
Branch: refs/heads/master
Commit: aa294cf6b2c03e3ab0816cd447b72f4c0ffee4aa
Parents: 8c59f0d
Author: Paul Rogers <pr...@cloudera.com>
Authored: Wed Nov 21 15:59:19 2018 -0800
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Thu Nov 22 04:36:21 2018 +0000
----------------------------------------------------------------------
.../apache/impala/util/treevis/AstPrinter.java | 47 ++++
.../apache/impala/util/treevis/TreePrinter.java | 114 +++++++++
.../apache/impala/util/treevis/Visualizer.java | 232 +++++++++++++++++++
3 files changed, 393 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/aa294cf6/fe/src/test/java/org/apache/impala/util/treevis/AstPrinter.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/util/treevis/AstPrinter.java b/fe/src/test/java/org/apache/impala/util/treevis/AstPrinter.java
new file mode 100644
index 0000000..4e509d7
--- /dev/null
+++ b/fe/src/test/java/org/apache/impala/util/treevis/AstPrinter.java
@@ -0,0 +1,47 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.util.treevis;
+
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+
+import org.apache.impala.analysis.Analyzer;
+import org.apache.impala.analysis.ParseNode;
+import org.apache.impala.analysis.TupleDescriptor;
+import org.apache.impala.catalog.HdfsTable;
+import org.apache.impala.catalog.ScalarType;
+
+/**
+ * Utility class to write a raw or decorated Impala AST
+ * to the console for debugging, with default formatting
+ * options.
+ */
+public class AstPrinter {
+
+ public static void printTree(ParseNode node) {
+ Visualizer vis = new Visualizer(
+ new TreePrinter(new PrintWriter(
+ new OutputStreamWriter(System.out),
+ true)));
+ vis.ignore(TupleDescriptor.class);
+ vis.ignore(HdfsTable.class);
+ vis.ignore(Analyzer.class);
+ vis.scalar(ScalarType.class);
+ vis.visualize(node);
+ }
+}
http://git-wip-us.apache.org/repos/asf/impala/blob/aa294cf6/fe/src/test/java/org/apache/impala/util/treevis/TreePrinter.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/util/treevis/TreePrinter.java b/fe/src/test/java/org/apache/impala/util/treevis/TreePrinter.java
new file mode 100644
index 0000000..b141741
--- /dev/null
+++ b/fe/src/test/java/org/apache/impala/util/treevis/TreePrinter.java
@@ -0,0 +1,114 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.util.treevis;
+
+import java.io.PrintWriter;
+import java.util.ArrayDeque;
+import java.util.Deque;
+
+import org.apache.impala.util.treevis.Visualizer.TreeVisualizer;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Rudimentary tree printer for use in debugging. Writes the
+ * result to stdout in a semi-JSON-like format. Emphasis is
+ * on human readability, not accurate serialization.
+ */
+public class TreePrinter implements TreeVisualizer {
+
+ private enum LevelType { OBJ, ARRAY }
+
+ private final PrintWriter out;
+ private final Deque<LevelType> levels = new ArrayDeque<>();
+
+ public TreePrinter(PrintWriter out) {
+ this.out = out;
+ }
+
+ @Override
+ public void startObj(String name, Object obj) {
+ indent();
+ printName(name);
+ out.print(" (");
+ out.print(obj.getClass().getSimpleName());
+ out.println( "): {");
+ levels.push(LevelType.OBJ);
+ }
+
+ private void printName(String name) {
+ name = name.replaceAll("_$", "");
+ out.print(name);
+ }
+
+ @Override
+ public void startArray(String name) {
+ indent();
+ printName(name);
+ out.println( ": [");
+ levels.push(LevelType.ARRAY);
+ }
+
+ @Override
+ public void field(String name, Object value) {
+ indent();
+ printName(name);
+ out.print(": ");
+ if (value == null) {
+ out.println("<null>");
+ } else if (value instanceof String) {
+ out.print("\"");
+ out.print(value);
+ out.println("\"");
+ } else {
+ out.println(value.toString());
+ }
+ }
+
+ @Override
+ public void endArray() {
+ Preconditions.checkState(! levels.isEmpty());
+ Preconditions.checkState(levels.peek() == LevelType.ARRAY);
+ levels.pop();
+ indent();
+ out.println("]");
+ }
+
+ @Override
+ public void endObj() {
+ Preconditions.checkState(! levels.isEmpty());
+ Preconditions.checkState(levels.peek() == LevelType.OBJ);
+ levels.pop();
+ indent();
+ out.println("}");
+ }
+
+ private void indent() {
+ for (int i = 0; i < levels.size(); i++) {
+ out.print(". ");
+ }
+ }
+
+ @Override
+ public void special(String name, String value) {
+ indent();
+ printName(name);
+ out.print(": ");
+ out.println(value);
+ }
+}
http://git-wip-us.apache.org/repos/asf/impala/blob/aa294cf6/fe/src/test/java/org/apache/impala/util/treevis/Visualizer.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/util/treevis/Visualizer.java b/fe/src/test/java/org/apache/impala/util/treevis/Visualizer.java
new file mode 100644
index 0000000..0b861ff
--- /dev/null
+++ b/fe/src/test/java/org/apache/impala/util/treevis/Visualizer.java
@@ -0,0 +1,232 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.util.treevis;
+
+import java.lang.reflect.Array;
+import java.lang.reflect.Field;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.IdentityHashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.impala.analysis.Expr;
+
+/**
+ * Visualizes a tree structure. Walks the tree, printing most
+ * fields of most objects. Excludes fields from the Object class,
+ * objects that would cause a cycle, and those specifically requested
+ * to ignored. Prints scalar-like values directly, else expands objects
+ * and array-like objects (including collections.)
+ *
+ * Objects can implement a method to do their own visualization:
+ *
+ * public void visualize(Visualizer vis);
+ *
+ * If this method exists, the visualizer calls it instead of walking
+ * the object's fields.
+ */
+public class Visualizer {
+
+ /**
+ * Performs actual visualization of the tree based on
+ * a set of events.
+ */
+ public interface TreeVisualizer {
+ void startObj(String name, Object obj);
+ void startArray(String name);
+ void field(String name, Object value);
+ void special(String name, String value);
+ void endArray();
+ void endObj();
+ }
+
+ private static final Class<?> STD_SCALARS[] = {
+ Byte.class,
+ Integer.class,
+ Character.class,
+ Long.class,
+ Float.class,
+ Double.class,
+ String.class,
+ Boolean.class,
+ Enum.class,
+ AtomicLong.class
+ };
+
+ private TreeVisualizer treeVis;
+ private Set<Class<?>> scalarTypes = new HashSet<>();
+ private Set<Class<?>> ignoreTypes = new HashSet<>();
+ // Since there is no IdentityHashMap. Values ignored.
+ private Map<Object, Object> parents = new IdentityHashMap<>();
+
+ public Visualizer(TreeVisualizer treeVis) {
+ this.treeVis = treeVis;
+ for (Class<?> c : STD_SCALARS) {
+ scalarTypes.add(c);
+ }
+ }
+
+ /**
+ * Specify a class to ignore. Objects of this type are
+ * skipped during visualization, with a special message
+ * for that field in place of object expansion.
+ *
+ * @param cls the class to skip
+ */
+ public void ignore(Class<?> cls) {
+ ignoreTypes.add(cls);
+ }
+
+ /**
+ * Specifies a scalar-like class to be rendered using a simple
+ * toString() call.
+ *
+ * @param cls the class to render using toString()
+ */
+ public void scalar(Class<?> cls) {
+ scalarTypes.add(cls);
+ }
+
+ /**
+ * Visualization entry point.
+ *
+ * @param root the object to visualize
+ */
+ public void visualize(Object root) {
+ treeVis.startObj("<root>", root);
+ visit(root);
+ treeVis.endObj();
+ }
+
+ public void visit(Object obj) {
+ Class<?> objClass = obj.getClass();
+ Method visMethod;
+ try {
+ visMethod = objClass.getMethod("visualize", getClass());
+ } catch (NoSuchMethodException e) {
+ visualizeObj(obj);
+ return;
+ } catch (SecurityException e) {
+ throw new IllegalStateException(e);
+ }
+ try {
+ visMethod.invoke(obj, this);
+ } catch (IllegalAccessException | IllegalArgumentException | InvocationTargetException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ private void visualizeObj(Object obj) {
+ if (obj instanceof Expr) {
+ System.out.print("");
+ }
+ Class<?> objClass = obj.getClass();
+ visualizeMembers(obj, objClass);
+ }
+
+ private void visualizeMembers(Object obj, Class<?> objClass) {
+ if (objClass == Object.class) return;
+ Class<?> parent = objClass.getSuperclass();
+ visualizeMembers(obj, parent);
+ Field[] fields = objClass.getDeclaredFields();
+ for (Field field : fields) {
+ if (Modifier.isStatic(field.getModifiers())) { continue; }
+ String name = field.getName();
+ try {
+ field.setAccessible(true);
+ Object value = field.get(obj);
+ visitValue(name, value);
+ } catch (IllegalArgumentException | IllegalAccessException e) {
+ treeVis.special(name, "<unavailable>");
+ }
+ }
+ }
+
+ public void visitValue(String name, Object value) {
+ if (value == null) {
+ treeVis.field(name, value);
+ return;
+ }
+ Class<?> valueClass = value.getClass();
+ if (valueClass.isArray()) {
+ visualizeArray(name, value);
+ return;
+ }
+ if (scalarTypes.contains(valueClass)) {
+ treeVis.field(name, value);
+ return;
+ }
+ if (ignoreTypes.contains(valueClass)) {
+ treeVis.special(name, "<Skip " + valueClass.getSimpleName() + ">");
+ return;
+ }
+ if (parents.containsKey(value)) {
+ treeVis.special(name, "<Back pointer to " + valueClass.getSimpleName() + ">");
+ return;
+ }
+ parents.put(value, value);
+ if (value instanceof Collection) {
+ visitCollection(name, (Collection<?>) value);
+ } else {
+ treeVis.startObj(name, value);
+ visit(value);
+ treeVis.endObj();
+ }
+ parents.remove(value);
+ }
+
+ private void visualizeArray(String name, Object value) {
+ // TODO: Specially handle scalar arrays
+ int len = Array.getLength(value);
+ if (len == 0) {
+ treeVis.special(name, "[]");
+ return;
+ }
+ treeVis.startArray(name);
+ for (int i = 0; i < len; i++) {
+ visitValue(Integer.toString(i), Array.get(value, i));
+ }
+ treeVis.endArray();
+ }
+
+ private void visitCollection(String name, Collection<?> col) {
+ if (col.isEmpty()) {
+ treeVis.special(name, "[]");
+ return;
+ }
+ treeVis.startArray(name);
+ if (col instanceof Map) {
+ Map<?, ?> map = (Map<?, ?>) col;
+ for (Entry<?, ?> entry : map.entrySet()) {
+ visitValue(entry.getKey().toString(), entry.getValue());
+ }
+ } else {
+ int i = 0;
+ for (Object member : col) {
+ visitValue(Integer.toString(i++), member);
+ }
+ }
+ treeVis.endArray();
+ }
+}