You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@calcite.apache.org by GitBox <gi...@apache.org> on 2021/08/15 17:11:57 UTC
[GitHub] [calcite] zuozhiw opened a new pull request #2486: [CALCITE-4737] Add Volcano Visualizer for Debugging
zuozhiw opened a new pull request #2486:
URL: https://github.com/apache/calcite/pull/2486
This pull request adds a new visualizer tool to visualize and debug the volcano planner optimization process.
Here are some gifs that shows some of its features:
The whole memo structure, including all RelSets, RelSubsets, and RelNodes are displayed:
A step-by-step process of each rule application is displayed. Click on each rule to see it's matched nodes (purple color), and newly generated nodes (cyan color).
See on the final best plan selected by the volcano optimizer. The cost of every RelNode is displayed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@calcite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [calcite] thomasrebele commented on a change in pull request #2486: [CALCITE-4737] Add Volcano Visualizer for Debugging
Posted by GitBox <gi...@apache.org>.
thomasrebele commented on a change in pull request #2486:
URL: https://github.com/apache/calcite/pull/2486#discussion_r689816076
##########
File path: core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRuleMatchVisualizer.java
##########
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.plan.volcano;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptCost;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.tools.visualizer.InputExcludedRelWriter;
+import org.apache.calcite.tools.visualizer.VisualizerNodeInfo;
+import org.apache.calcite.tools.visualizer.VisualizerRuleMatchInfo;
+
+import org.apache.commons.io.IOUtils;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Charsets;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+import static java.util.stream.Collectors.joining;
+
+/**
+ * This is tool to visualize the rule match process of the VolcanoPlanner.
+ *
+ *
+ * <p>To use the visualizer, add a listener before the VolcanoPlanner optimization phase.
+ * Then writes the output to a file after the optimization ends.
+ *
+ * <pre>
+ * // construct the visualizer and attach a listener to VolcanoPlanner
+ * VolcanoRuleMatchVisualizerListener visualizerListener =
+ * new VolcanoRuleMatchVisualizerListener(volcanoPlanner);
+ * volcanoPlanner.addListener(visualizerListener);
+ *
+ * volcanoPlanner.findBestExpr();
+ *
+ * // after the optimization, adds the final best plan
+ * visualizerListener.getVisualizer().addFinalPlan();
+ * // writes the output to files
+ * visualizerListener.getVisualizer().writeToFile(outputDirectory, "");
+ * </pre>
+ */
+public class VolcanoRuleMatchVisualizer {
+
+ VolcanoPlanner volcanoPlanner;
+
+ // a sequence of ruleMatch ID to represent the order of rule match
+ List<String> ruleMatchSequence = new ArrayList<>();
+ // map of ruleMatch ID and the info, including the state snapshot at the time of ruleMatch
+ Map<String, VisualizerRuleMatchInfo> ruleInfoMap = new HashMap<>();
+ // map of nodeID to the ruleID it's first added
+ Map<String, String> nodeAddedInRule = new HashMap<>();
+
+ // a map of relNode ID to the actual RelNode object
+ // contains all the relNodes appear during the optimization
+ // all RelNode are immutable in Calcite, therefore only new nodes will be added
+ Map<String, RelNode> allNodes = new HashMap<>();
+
+ public VolcanoRuleMatchVisualizer(VolcanoPlanner volcanoPlanner) {
+ this.volcanoPlanner = volcanoPlanner;
+ }
+
+ public void addRuleMatch(String ruleCallID, Collection<? extends RelNode> matchedRels) {
+
+ // store the current state snapshot
+ // nodes contained in the sets
+ // and inputs of relNodes (and relSubsets)
+ Map<String, String> setLabels = new HashMap<>();
+ Map<String, String> setOriginalRel = new HashMap<>();
+ Map<String, Set<String>> nodesInSet = new HashMap<>();
+ Map<String, Set<String>> nodeInputs = new HashMap<>();
+
+ // newNodes appeared after this ruleCall
+ Set<String> newNodes = new HashSet<>();
+
+ // populate current snapshot, and fill in the allNodes map
+ volcanoPlanner.allSets.forEach(set -> {
+ String setID = "set-" + set.id;
+ String setLabel = getSetLabel(set);
+ setLabels.put(setID, setLabel);
+ setOriginalRel.put(setID, set.rel == null ? "" : String.valueOf(set.rel.getId()));
+
+ nodesInSet.put(setID, nodesInSet.getOrDefault(setID, new HashSet<>()));
+
+ Consumer<RelNode> addNode = rel -> {
+ String nodeID = String.valueOf(rel.getId());
+ nodesInSet.get(setID).add(nodeID);
+
+ if (!allNodes.containsKey(nodeID)) {
+ newNodes.add(nodeID);
+ allNodes.put(nodeID, rel);
+ }
+ };
+
+ Consumer<RelNode> addLink = rel -> {
+ String nodeID = String.valueOf(rel.getId());
+ nodeInputs.put(nodeID, new HashSet<>());
+ if (rel instanceof RelSubset) {
+ RelSubset relSubset = (RelSubset) rel;
+ relSubset.getRelList().stream()
+ .filter(input -> input.getTraitSet().equals(relSubset.getTraitSet()))
+ .forEach(input -> nodeInputs.get(nodeID).add(String.valueOf(input.getId())));
Review comment:
Thanks for the clarification. It's better to reduce the number of arrows in the graph, so let's keep it as it is.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@calcite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [calcite] zuozhiw commented on a change in pull request #2486: [CALCITE-4737] Add Volcano Visualizer for Debugging
Posted by GitBox <gi...@apache.org>.
zuozhiw commented on a change in pull request #2486:
URL: https://github.com/apache/calcite/pull/2486#discussion_r689666598
##########
File path: core/src/main/resources/volcano-viz/volcano-viz-data.js
##########
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// sample generated visualizer data used by the visualizer JavaScript code
+var data = {
+
+ allNodes: {
+ "node1": {
+ label: "TableScan-1", // node label being displayed
+ isSubset: false, // if the node is a RelSubset
+ explanation: "table=t1", // additional explanation of properties
+ finalCost: "100 cpu, 100io" // final cost (in string) of the node at the end of optimization
+ },
+ "node2": {
+ label: "Filter-2",
+ isSubset: false,
+ explanation: "condition=c",
+ finalCost: "200cpu, 100io"
+ },
+ "node3": {
+ label: "TableSink-3",
+ isSubset: false,
+ explanation: "table=t2",
+ finalCost: "20cpu, 20io"
+ },
+ "node4": {
+ label: "IndexTableScan-4",
+ isSubset: false,
+ explanation: "table=t1, condition=c",
+ finalCost: "10cpu, 10io"
+ },
+ },
+
+ ruleMatchSequence: [
+ "INITIAL",
+ "IndexTableScanRule#1",
+ "FINAL"
+ ],
+
+ ruleMatchInfoMap: {
+ "INITIAL": {
+ setLabels: {
+ "set1": "set1",
+ "set2": "set2",
+ "set3": "set3",
+ },
+ setOriginalRel: {
+ "set1": "node1",
+ "set2": "node2",
+ "set3": "node3",
+ },
Review comment:
agree, this can make the json more compact. I can do the JS changes since I'm more familiar.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@calcite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [calcite] zuozhiw commented on a change in pull request #2486: [CALCITE-4737] Add Volcano Visualizer for Debugging
Posted by GitBox <gi...@apache.org>.
zuozhiw commented on a change in pull request #2486:
URL: https://github.com/apache/calcite/pull/2486#discussion_r689650606
##########
File path: core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRuleMatchVisualizer.java
##########
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.plan.volcano;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptCost;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.tools.visualizer.InputExcludedRelWriter;
+import org.apache.calcite.tools.visualizer.VisualizerNodeInfo;
+import org.apache.calcite.tools.visualizer.VisualizerRuleMatchInfo;
+
+import org.apache.commons.io.IOUtils;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Charsets;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+import static java.util.stream.Collectors.joining;
+
+/**
+ * This is tool to visualize the rule match process of the VolcanoPlanner.
+ *
+ *
+ * <p>To use the visualizer, add a listener before the VolcanoPlanner optimization phase.
+ * Then writes the output to a file after the optimization ends.
+ *
+ * <pre>
+ * // construct the visualizer and attach a listener to VolcanoPlanner
+ * VolcanoRuleMatchVisualizerListener visualizerListener =
+ * new VolcanoRuleMatchVisualizerListener(volcanoPlanner);
+ * volcanoPlanner.addListener(visualizerListener);
+ *
+ * volcanoPlanner.findBestExpr();
+ *
+ * // after the optimization, adds the final best plan
+ * visualizerListener.getVisualizer().addFinalPlan();
+ * // writes the output to files
+ * visualizerListener.getVisualizer().writeToFile(outputDirectory, "");
+ * </pre>
+ */
+public class VolcanoRuleMatchVisualizer {
+
+ VolcanoPlanner volcanoPlanner;
+
+ // a sequence of ruleMatch ID to represent the order of rule match
+ List<String> ruleMatchSequence = new ArrayList<>();
+ // map of ruleMatch ID and the info, including the state snapshot at the time of ruleMatch
+ Map<String, VisualizerRuleMatchInfo> ruleInfoMap = new HashMap<>();
+ // map of nodeID to the ruleID it's first added
+ Map<String, String> nodeAddedInRule = new HashMap<>();
+
+ // a map of relNode ID to the actual RelNode object
+ // contains all the relNodes appear during the optimization
+ // all RelNode are immutable in Calcite, therefore only new nodes will be added
Review comment:
I think we can rephrase it. Although a RelNode's children can be modified, its own properties will not change. I was mainly refering to the immutable properties of a relnode.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@calcite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [calcite] zuozhiw closed pull request #2486: [CALCITE-4737] Add Volcano Visualizer for Debugging
Posted by GitBox <gi...@apache.org>.
zuozhiw closed pull request #2486:
URL: https://github.com/apache/calcite/pull/2486
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@calcite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [calcite] zuozhiw commented on a change in pull request #2486: [CALCITE-4737] Add Volcano Visualizer for Debugging
Posted by GitBox <gi...@apache.org>.
zuozhiw commented on a change in pull request #2486:
URL: https://github.com/apache/calcite/pull/2486#discussion_r689659368
##########
File path: core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRuleMatchVisualizer.java
##########
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.plan.volcano;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptCost;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.tools.visualizer.InputExcludedRelWriter;
+import org.apache.calcite.tools.visualizer.VisualizerNodeInfo;
+import org.apache.calcite.tools.visualizer.VisualizerRuleMatchInfo;
+
+import org.apache.commons.io.IOUtils;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Charsets;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+import static java.util.stream.Collectors.joining;
+
+/**
+ * This is tool to visualize the rule match process of the VolcanoPlanner.
+ *
+ *
+ * <p>To use the visualizer, add a listener before the VolcanoPlanner optimization phase.
+ * Then writes the output to a file after the optimization ends.
+ *
+ * <pre>
+ * // construct the visualizer and attach a listener to VolcanoPlanner
+ * VolcanoRuleMatchVisualizerListener visualizerListener =
+ * new VolcanoRuleMatchVisualizerListener(volcanoPlanner);
+ * volcanoPlanner.addListener(visualizerListener);
Review comment:
agree, I saw the changes in your branch and the api is more concise and easy to use.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@calcite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [calcite] zuozhiw commented on a change in pull request #2486: [CALCITE-4737] Add Volcano Visualizer for Debugging
Posted by GitBox <gi...@apache.org>.
zuozhiw commented on a change in pull request #2486:
URL: https://github.com/apache/calcite/pull/2486#discussion_r689650805
##########
File path: core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRuleMatchVisualizer.java
##########
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.plan.volcano;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptCost;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.tools.visualizer.InputExcludedRelWriter;
+import org.apache.calcite.tools.visualizer.VisualizerNodeInfo;
+import org.apache.calcite.tools.visualizer.VisualizerRuleMatchInfo;
+
+import org.apache.commons.io.IOUtils;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Charsets;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+import static java.util.stream.Collectors.joining;
+
+/**
+ * This is tool to visualize the rule match process of the VolcanoPlanner.
+ *
+ *
+ * <p>To use the visualizer, add a listener before the VolcanoPlanner optimization phase.
+ * Then writes the output to a file after the optimization ends.
+ *
+ * <pre>
+ * // construct the visualizer and attach a listener to VolcanoPlanner
+ * VolcanoRuleMatchVisualizerListener visualizerListener =
+ * new VolcanoRuleMatchVisualizerListener(volcanoPlanner);
+ * volcanoPlanner.addListener(visualizerListener);
+ *
+ * volcanoPlanner.findBestExpr();
+ *
+ * // after the optimization, adds the final best plan
+ * visualizerListener.getVisualizer().addFinalPlan();
+ * // writes the output to files
+ * visualizerListener.getVisualizer().writeToFile(outputDirectory, "");
+ * </pre>
+ */
+public class VolcanoRuleMatchVisualizer {
+
+ VolcanoPlanner volcanoPlanner;
+
+ // a sequence of ruleMatch ID to represent the order of rule match
+ List<String> ruleMatchSequence = new ArrayList<>();
+ // map of ruleMatch ID and the info, including the state snapshot at the time of ruleMatch
+ Map<String, VisualizerRuleMatchInfo> ruleInfoMap = new HashMap<>();
+ // map of nodeID to the ruleID it's first added
+ Map<String, String> nodeAddedInRule = new HashMap<>();
+
+ // a map of relNode ID to the actual RelNode object
+ // contains all the relNodes appear during the optimization
+ // all RelNode are immutable in Calcite, therefore only new nodes will be added
+ Map<String, RelNode> allNodes = new HashMap<>();
+
+ public VolcanoRuleMatchVisualizer(VolcanoPlanner volcanoPlanner) {
+ this.volcanoPlanner = volcanoPlanner;
+ }
+
+ public void addRuleMatch(String ruleCallID, Collection<? extends RelNode> matchedRels) {
+
+ // store the current state snapshot
+ // nodes contained in the sets
+ // and inputs of relNodes (and relSubsets)
+ Map<String, String> setLabels = new HashMap<>();
+ Map<String, String> setOriginalRel = new HashMap<>();
+ Map<String, Set<String>> nodesInSet = new HashMap<>();
+ Map<String, Set<String>> nodeInputs = new HashMap<>();
+
+ // newNodes appeared after this ruleCall
+ Set<String> newNodes = new HashSet<>();
+
+ // populate current snapshot, and fill in the allNodes map
+ volcanoPlanner.allSets.forEach(set -> {
+ String setID = "set-" + set.id;
+ String setLabel = getSetLabel(set);
+ setLabels.put(setID, setLabel);
+ setOriginalRel.put(setID, set.rel == null ? "" : String.valueOf(set.rel.getId()));
+
+ nodesInSet.put(setID, nodesInSet.getOrDefault(setID, new HashSet<>()));
+
+ Consumer<RelNode> addNode = rel -> {
+ String nodeID = String.valueOf(rel.getId());
+ nodesInSet.get(setID).add(nodeID);
+
+ if (!allNodes.containsKey(nodeID)) {
+ newNodes.add(nodeID);
+ allNodes.put(nodeID, rel);
+ }
+ };
+
+ Consumer<RelNode> addLink = rel -> {
+ String nodeID = String.valueOf(rel.getId());
+ nodeInputs.put(nodeID, new HashSet<>());
Review comment:
agree
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@calcite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [calcite] zuozhiw commented on a change in pull request #2486: [CALCITE-4737] Add Volcano Visualizer for Debugging
Posted by GitBox <gi...@apache.org>.
zuozhiw commented on a change in pull request #2486:
URL: https://github.com/apache/calcite/pull/2486#discussion_r689660473
##########
File path: core/src/main/resources/volcano-viz/viz-template.html
##########
@@ -0,0 +1,323 @@
+<!doctype html>
+<html lang="en">
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+<meta charset="utf-8">
+<title>Calcite Rule Match Visualization</title>
+
+<script src="https://d3js.org/d3.v5.min.js" charset="utf-8"></script>
+<script src="https://dagrejs.github.io/project/dagre-d3/latest/dagre-d3.min.js"></script>
+<script src="https://d3js.org/d3-zoom.v1.min.js"></script>
+<script src="https://unpkg.com/tippy.js@3/dist/tippy.all.min.js"></script>
+<script src="volcano-viz-data.js"></script>
+
+<style id="css">
+ body {
+ height: 100%;
+ margin: 0 0;
+ color: #333;
+ font-weight: 300;
+ font-family: "Helvetica Neue", Helvetica, Arial, sans-serf;
+ }
+
+ li a {
+ display: block;
+ /* and you can use padding for additional space if needs, as a clickable area / or other styling */
+ padding: 5px 20px;
+ }
+
+ section {
+ margin-bottom: 3em;
+ }
+
+ section p {
+ text-align: justify;
+ }
+
+ svg {
+ border: 1px solid #ccc;
+ overflow: hidden;
+ margin: 0 auto;
+ }
+
+ pre {
+ border: 1px solid #ccc;
+ }
+
+ .clusters rect {
+ fill: #FFFFE0;
+ stroke: #999;
+ stroke-width: 1.5px;
+ }
+
+ text {
+ font-weight: 300;
+ font-family: "Helvetica Neue", Helvetica, Arial, sans-serf;
+ font-size: 2em;
+ }
+
+ .node rect {
+ stroke: #999;
+ fill: #fff;
+ stroke-width: 1.5px;
+ }
+
+ .edgePath path {
+ stroke: #333;
+ stroke-width: 2px;
+ }
+
+ .container {
+ display: flex;
+ align-items: center;
+ }
+
+ .column1 {
+ flex: 0 0 300px;
+ }
+
+ .column2 {
+ flex: 0 0 1000px;
+ }
+
+ .tippy-content {
+ word-break: break-all;
+ word-wrap: break-word;
+ }
+</style>
+
+<div class="container">
+ <div class="column1">
+ <div style="width: 100%; text-align: center">
+ <div id="current-rule" style="display: block"></div>
+ <button id="prev-button" style="width: 80px; height: 40px; display:inline-block" disabled>prev </button>
+ <button id="next-button" style="width: 80px; height: 40px; display:inline-block" disabled> next</button>
+ </div>
+ <ul id="rule-match-list" style="width: 300px; height: 600px; overflow: auto">
+ </ul>
+ </div>
+ <div class="column2">
+ <svg id="svg-canvas" width="1200px" height="800px"></svg>
+ </div>
+</div>
+
+<script id="js">
+
+ var allNodes = data.allNodes;
+ var ruleMatchSequence = data.ruleMatchSequence;
+ var ruleMatchInfoMap = data.ruleMatchInfoMap;
+ var nodeAddedInRule = data.nodeAddedInRule;
+
+ /*
+ * Graph data and D3 JS render related variables
+ */
+
+ // Create the input graph
+ var g = new dagreD3.graphlib.Graph({
+ compound: true
+ })
+ .setGraph({
+ rankdir: 'LR'
+ })
+ .setDefaultEdgeLabel(function () {
+ return {};
+ });
+
+ // Create the renderer
+ var render = new dagreD3.render();
+
+ // Set up an SVG group so that we can translate the final graph.
+ var svg = d3.select("svg");
+ var svgGroup = svg.append("g");
+
+ // Set up zoom support
+ var svg = d3.select("svg")
+ .attr("width", "1200px")
+ .attr("height", "800px")
+ .call(d3.zoom().on("zoom", function () {
+ svgGroup.attr("transform", d3.event.transform)
+ }));
+
+ /*
+ * Global State
+ */
+
+ var currentRuleID = undefined;
+
+ /*
+ * Event Handler functions
+ */
+
+ var setCurrentRule = (ruleMatchID) => {
+ // un-highlight previous entry
+ var prevRuleID = currentRuleID;
+ if (prevRuleID !== undefined) {
+ var prevRuleElement = document.getElementById(prevRuleID);
+ prevRuleElement.style.backgroundColor = "#FFFFFF";
+ }
+
+ currentRuleID = ruleMatchID;
+ document.getElementById('current-rule').innerText = currentRuleID;
+
+ var currentRuleElement = document.getElementById(currentRuleID);
+ currentRuleElement.style.backgroundColor = "#D3D3D3";
+
+ var ruleIndex = ruleMatchSequence.indexOf(currentRuleID);
+
+ document.getElementById("prev-button").disabled = false;
+ document.getElementById("next-button").disabled = false;
Review comment:
agree
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@calcite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [calcite] thomasrebele commented on a change in pull request #2486: [CALCITE-4737] Add Volcano Visualizer for Debugging
Posted by GitBox <gi...@apache.org>.
thomasrebele commented on a change in pull request #2486:
URL: https://github.com/apache/calcite/pull/2486#discussion_r689829370
##########
File path: core/src/main/resources/volcano-viz/viz-template.html
##########
@@ -0,0 +1,323 @@
+<!doctype html>
+<html lang="en">
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+<meta charset="utf-8">
+<title>Calcite Rule Match Visualization</title>
+
+<script src="https://d3js.org/d3.v5.min.js" charset="utf-8"></script>
+<script src="https://dagrejs.github.io/project/dagre-d3/latest/dagre-d3.min.js"></script>
+<script src="https://d3js.org/d3-zoom.v1.min.js"></script>
+<script src="https://unpkg.com/tippy.js@3/dist/tippy.all.min.js"></script>
+<script src="volcano-viz-data.js"></script>
+
+<style id="css">
+ body {
+ height: 100%;
+ margin: 0 0;
+ color: #333;
+ font-weight: 300;
+ font-family: "Helvetica Neue", Helvetica, Arial, sans-serf;
+ }
+
+ li a {
+ display: block;
+ /* and you can use padding for additional space if needs, as a clickable area / or other styling */
+ padding: 5px 20px;
+ }
+
+ section {
+ margin-bottom: 3em;
+ }
+
+ section p {
+ text-align: justify;
+ }
+
+ svg {
+ border: 1px solid #ccc;
+ overflow: hidden;
+ margin: 0 auto;
+ }
+
+ pre {
+ border: 1px solid #ccc;
+ }
+
+ .clusters rect {
+ fill: #FFFFE0;
+ stroke: #999;
+ stroke-width: 1.5px;
+ }
+
+ text {
+ font-weight: 300;
+ font-family: "Helvetica Neue", Helvetica, Arial, sans-serf;
+ font-size: 2em;
+ }
+
+ .node rect {
+ stroke: #999;
+ fill: #fff;
+ stroke-width: 1.5px;
+ }
+
+ .edgePath path {
+ stroke: #333;
+ stroke-width: 2px;
+ }
+
+ .container {
+ display: flex;
+ align-items: center;
+ }
+
+ .column1 {
+ flex: 0 0 300px;
+ }
+
+ .column2 {
+ flex: 0 0 1000px;
+ }
+
+ .tippy-content {
+ word-break: break-all;
+ word-wrap: break-word;
+ }
+</style>
+
+<div class="container">
+ <div class="column1">
+ <div style="width: 100%; text-align: center">
+ <div id="current-rule" style="display: block"></div>
+ <button id="prev-button" style="width: 80px; height: 40px; display:inline-block" disabled>prev </button>
+ <button id="next-button" style="width: 80px; height: 40px; display:inline-block" disabled> next</button>
+ </div>
+ <ul id="rule-match-list" style="width: 300px; height: 600px; overflow: auto">
+ </ul>
+ </div>
+ <div class="column2">
+ <svg id="svg-canvas" width="1200px" height="800px"></svg>
+ </div>
+</div>
+
+<script id="js">
+
+ var allNodes = data.allNodes;
+ var ruleMatchSequence = data.ruleMatchSequence;
+ var ruleMatchInfoMap = data.ruleMatchInfoMap;
+ var nodeAddedInRule = data.nodeAddedInRule;
+
+ /*
+ * Graph data and D3 JS render related variables
+ */
+
+ // Create the input graph
+ var g = new dagreD3.graphlib.Graph({
+ compound: true
+ })
+ .setGraph({
+ rankdir: 'LR'
+ })
+ .setDefaultEdgeLabel(function () {
+ return {};
+ });
+
+ // Create the renderer
+ var render = new dagreD3.render();
+
+ // Set up an SVG group so that we can translate the final graph.
+ var svg = d3.select("svg");
+ var svgGroup = svg.append("g");
+
+ // Set up zoom support
+ var svg = d3.select("svg")
+ .attr("width", "1200px")
+ .attr("height", "800px")
+ .call(d3.zoom().on("zoom", function () {
+ svgGroup.attr("transform", d3.event.transform)
+ }));
+
+ /*
+ * Global State
+ */
+
+ var currentRuleID = undefined;
+
+ /*
+ * Event Handler functions
+ */
+
+ var setCurrentRule = (ruleMatchID) => {
+ // un-highlight previous entry
+ var prevRuleID = currentRuleID;
+ if (prevRuleID !== undefined) {
+ var prevRuleElement = document.getElementById(prevRuleID);
+ prevRuleElement.style.backgroundColor = "#FFFFFF";
+ }
+
+ currentRuleID = ruleMatchID;
+ document.getElementById('current-rule').innerText = currentRuleID;
+
+ var currentRuleElement = document.getElementById(currentRuleID);
+ currentRuleElement.style.backgroundColor = "#D3D3D3";
+
+ var ruleIndex = ruleMatchSequence.indexOf(currentRuleID);
+
+ document.getElementById("prev-button").disabled = false;
+ document.getElementById("next-button").disabled = false;
+
+ if (ruleIndex === 0) {
+ document.getElementById("prev-button").disabled = true;
+ }
+ if (ruleIndex === ruleMatchSequence.length - 1) {
+ document.getElementById("next-button").disabled = true;
+ }
+
+ createGraph(ruleMatchID);
+ }
+
+ var createGraph = (ruleMatchID) => {
+ var ruleMatchInfo = ruleMatchInfoMap[ruleMatchID]
+ console.log(ruleMatchInfo);
+
+ // remove previous rendered view and clear graph model
+ d3.select("svg g").selectAll("*").remove();
+ g.nodes().slice().forEach(nodeID => g.removeNode(nodeID));
Review comment:
Indeed, merging sets would be quite difficult to handle.
How big is an ulta-large plan? Maybe we can provide a combo-box to switch between different layout algorithms (e.g., d3-force).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@calcite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [calcite] zuozhiw commented on a change in pull request #2486: [CALCITE-4737] Add Volcano Visualizer for Debugging
Posted by GitBox <gi...@apache.org>.
zuozhiw commented on a change in pull request #2486:
URL: https://github.com/apache/calcite/pull/2486#discussion_r689659568
##########
File path: core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRuleMatchVisualizer.java
##########
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.plan.volcano;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptCost;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.tools.visualizer.InputExcludedRelWriter;
+import org.apache.calcite.tools.visualizer.VisualizerNodeInfo;
+import org.apache.calcite.tools.visualizer.VisualizerRuleMatchInfo;
+
+import org.apache.commons.io.IOUtils;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Charsets;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+import static java.util.stream.Collectors.joining;
+
+/**
+ * This is tool to visualize the rule match process of the VolcanoPlanner.
+ *
+ *
+ * <p>To use the visualizer, add a listener before the VolcanoPlanner optimization phase.
+ * Then writes the output to a file after the optimization ends.
+ *
+ * <pre>
+ * // construct the visualizer and attach a listener to VolcanoPlanner
+ * VolcanoRuleMatchVisualizerListener visualizerListener =
+ * new VolcanoRuleMatchVisualizerListener(volcanoPlanner);
+ * volcanoPlanner.addListener(visualizerListener);
+ *
+ * volcanoPlanner.findBestExpr();
+ *
+ * // after the optimization, adds the final best plan
+ * visualizerListener.getVisualizer().addFinalPlan();
Review comment:
agree, makes it easy to use
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@calcite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [calcite] zuozhiw commented on a change in pull request #2486: [CALCITE-4737] Add Volcano Visualizer for Debugging
Posted by GitBox <gi...@apache.org>.
zuozhiw commented on a change in pull request #2486:
URL: https://github.com/apache/calcite/pull/2486#discussion_r689658918
##########
File path: core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRuleMatchVisualizer.java
##########
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.plan.volcano;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptCost;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.tools.visualizer.InputExcludedRelWriter;
+import org.apache.calcite.tools.visualizer.VisualizerNodeInfo;
+import org.apache.calcite.tools.visualizer.VisualizerRuleMatchInfo;
+
+import org.apache.commons.io.IOUtils;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Charsets;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+import static java.util.stream.Collectors.joining;
+
+/**
+ * This is tool to visualize the rule match process of the VolcanoPlanner.
+ *
+ *
+ * <p>To use the visualizer, add a listener before the VolcanoPlanner optimization phase.
+ * Then writes the output to a file after the optimization ends.
+ *
+ * <pre>
+ * // construct the visualizer and attach a listener to VolcanoPlanner
+ * VolcanoRuleMatchVisualizerListener visualizerListener =
+ * new VolcanoRuleMatchVisualizerListener(volcanoPlanner);
+ * volcanoPlanner.addListener(visualizerListener);
+ *
+ * volcanoPlanner.findBestExpr();
+ *
+ * // after the optimization, adds the final best plan
+ * visualizerListener.getVisualizer().addFinalPlan();
+ * // writes the output to files
+ * visualizerListener.getVisualizer().writeToFile(outputDirectory, "");
+ * </pre>
+ */
+public class VolcanoRuleMatchVisualizer {
+
+ VolcanoPlanner volcanoPlanner;
+
+ // a sequence of ruleMatch ID to represent the order of rule match
+ List<String> ruleMatchSequence = new ArrayList<>();
+ // map of ruleMatch ID and the info, including the state snapshot at the time of ruleMatch
+ Map<String, VisualizerRuleMatchInfo> ruleInfoMap = new HashMap<>();
+ // map of nodeID to the ruleID it's first added
+ Map<String, String> nodeAddedInRule = new HashMap<>();
+
+ // a map of relNode ID to the actual RelNode object
+ // contains all the relNodes appear during the optimization
+ // all RelNode are immutable in Calcite, therefore only new nodes will be added
+ Map<String, RelNode> allNodes = new HashMap<>();
+
+ public VolcanoRuleMatchVisualizer(VolcanoPlanner volcanoPlanner) {
+ this.volcanoPlanner = volcanoPlanner;
+ }
+
+ public void addRuleMatch(String ruleCallID, Collection<? extends RelNode> matchedRels) {
+
+ // store the current state snapshot
+ // nodes contained in the sets
+ // and inputs of relNodes (and relSubsets)
+ Map<String, String> setLabels = new HashMap<>();
+ Map<String, String> setOriginalRel = new HashMap<>();
+ Map<String, Set<String>> nodesInSet = new HashMap<>();
+ Map<String, Set<String>> nodeInputs = new HashMap<>();
+
+ // newNodes appeared after this ruleCall
+ Set<String> newNodes = new HashSet<>();
+
+ // populate current snapshot, and fill in the allNodes map
+ volcanoPlanner.allSets.forEach(set -> {
+ String setID = "set-" + set.id;
+ String setLabel = getSetLabel(set);
+ setLabels.put(setID, setLabel);
+ setOriginalRel.put(setID, set.rel == null ? "" : String.valueOf(set.rel.getId()));
+
+ nodesInSet.put(setID, nodesInSet.getOrDefault(setID, new HashSet<>()));
+
+ Consumer<RelNode> addNode = rel -> {
+ String nodeID = String.valueOf(rel.getId());
+ nodesInSet.get(setID).add(nodeID);
+
+ if (!allNodes.containsKey(nodeID)) {
+ newNodes.add(nodeID);
+ allNodes.put(nodeID, rel);
+ }
+ };
+
+ Consumer<RelNode> addLink = rel -> {
+ String nodeID = String.valueOf(rel.getId());
+ nodeInputs.put(nodeID, new HashSet<>());
+ if (rel instanceof RelSubset) {
+ RelSubset relSubset = (RelSubset) rel;
+ relSubset.getRelList().stream()
+ .filter(input -> input.getTraitSet().equals(relSubset.getTraitSet()))
+ .forEach(input -> nodeInputs.get(nodeID).add(String.valueOf(input.getId())));
+ relSubset.set.subsets.stream()
+ .filter(other -> !other.equals(relSubset))
+ .filter(other -> other.getTraitSet().satisfies(relSubset.getTraitSet()))
+ .forEach(other -> nodeInputs.get(nodeID).add(String.valueOf(other.getId())));
+ } else {
+ rel.getInputs().forEach(input -> nodeInputs.get(nodeID)
+ .add(String.valueOf(input.getId())));
+ }
+ };
+
+ set.rels.forEach(addNode);
+ set.subsets.forEach(addNode);
+ set.rels.forEach(addLink);
+ set.subsets.forEach(addLink);
+ });
+
+ // get the matched nodes of this rule
+ Set<String> matchedNodeIDs = matchedRels.stream()
+ .map(rel -> String.valueOf(rel.getId()))
+ .collect(Collectors.toSet());
+
+ // get importance 0 rels as of right now
+ Set<String> importanceZeroNodes = new HashSet<>();
Review comment:
agree
##########
File path: core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRuleMatchVisualizer.java
##########
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.plan.volcano;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptCost;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.tools.visualizer.InputExcludedRelWriter;
+import org.apache.calcite.tools.visualizer.VisualizerNodeInfo;
+import org.apache.calcite.tools.visualizer.VisualizerRuleMatchInfo;
+
+import org.apache.commons.io.IOUtils;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Charsets;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+import static java.util.stream.Collectors.joining;
+
+/**
+ * This is tool to visualize the rule match process of the VolcanoPlanner.
+ *
+ *
+ * <p>To use the visualizer, add a listener before the VolcanoPlanner optimization phase.
+ * Then writes the output to a file after the optimization ends.
+ *
+ * <pre>
+ * // construct the visualizer and attach a listener to VolcanoPlanner
+ * VolcanoRuleMatchVisualizerListener visualizerListener =
+ * new VolcanoRuleMatchVisualizerListener(volcanoPlanner);
+ * volcanoPlanner.addListener(visualizerListener);
+ *
+ * volcanoPlanner.findBestExpr();
+ *
+ * // after the optimization, adds the final best plan
+ * visualizerListener.getVisualizer().addFinalPlan();
+ * // writes the output to files
+ * visualizerListener.getVisualizer().writeToFile(outputDirectory, "");
+ * </pre>
+ */
+public class VolcanoRuleMatchVisualizer {
+
+ VolcanoPlanner volcanoPlanner;
+
+ // a sequence of ruleMatch ID to represent the order of rule match
+ List<String> ruleMatchSequence = new ArrayList<>();
+ // map of ruleMatch ID and the info, including the state snapshot at the time of ruleMatch
+ Map<String, VisualizerRuleMatchInfo> ruleInfoMap = new HashMap<>();
+ // map of nodeID to the ruleID it's first added
+ Map<String, String> nodeAddedInRule = new HashMap<>();
+
+ // a map of relNode ID to the actual RelNode object
+ // contains all the relNodes appear during the optimization
+ // all RelNode are immutable in Calcite, therefore only new nodes will be added
+ Map<String, RelNode> allNodes = new HashMap<>();
+
+ public VolcanoRuleMatchVisualizer(VolcanoPlanner volcanoPlanner) {
+ this.volcanoPlanner = volcanoPlanner;
+ }
+
+ public void addRuleMatch(String ruleCallID, Collection<? extends RelNode> matchedRels) {
+
+ // store the current state snapshot
+ // nodes contained in the sets
+ // and inputs of relNodes (and relSubsets)
+ Map<String, String> setLabels = new HashMap<>();
+ Map<String, String> setOriginalRel = new HashMap<>();
+ Map<String, Set<String>> nodesInSet = new HashMap<>();
+ Map<String, Set<String>> nodeInputs = new HashMap<>();
Review comment:
agree
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@calcite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [calcite] zuozhiw commented on a change in pull request #2486: [CALCITE-4737] Add Volcano Visualizer for Debugging
Posted by GitBox <gi...@apache.org>.
zuozhiw commented on a change in pull request #2486:
URL: https://github.com/apache/calcite/pull/2486#discussion_r689656891
##########
File path: core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRuleMatchVisualizer.java
##########
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.plan.volcano;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptCost;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.tools.visualizer.InputExcludedRelWriter;
+import org.apache.calcite.tools.visualizer.VisualizerNodeInfo;
+import org.apache.calcite.tools.visualizer.VisualizerRuleMatchInfo;
+
+import org.apache.commons.io.IOUtils;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Charsets;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+import static java.util.stream.Collectors.joining;
+
+/**
+ * This is tool to visualize the rule match process of the VolcanoPlanner.
+ *
+ *
+ * <p>To use the visualizer, add a listener before the VolcanoPlanner optimization phase.
+ * Then writes the output to a file after the optimization ends.
+ *
+ * <pre>
+ * // construct the visualizer and attach a listener to VolcanoPlanner
+ * VolcanoRuleMatchVisualizerListener visualizerListener =
+ * new VolcanoRuleMatchVisualizerListener(volcanoPlanner);
+ * volcanoPlanner.addListener(visualizerListener);
+ *
+ * volcanoPlanner.findBestExpr();
+ *
+ * // after the optimization, adds the final best plan
+ * visualizerListener.getVisualizer().addFinalPlan();
+ * // writes the output to files
+ * visualizerListener.getVisualizer().writeToFile(outputDirectory, "");
+ * </pre>
+ */
+public class VolcanoRuleMatchVisualizer {
+
+ VolcanoPlanner volcanoPlanner;
+
+ // a sequence of ruleMatch ID to represent the order of rule match
+ List<String> ruleMatchSequence = new ArrayList<>();
+ // map of ruleMatch ID and the info, including the state snapshot at the time of ruleMatch
+ Map<String, VisualizerRuleMatchInfo> ruleInfoMap = new HashMap<>();
+ // map of nodeID to the ruleID it's first added
+ Map<String, String> nodeAddedInRule = new HashMap<>();
+
+ // a map of relNode ID to the actual RelNode object
+ // contains all the relNodes appear during the optimization
+ // all RelNode are immutable in Calcite, therefore only new nodes will be added
+ Map<String, RelNode> allNodes = new HashMap<>();
+
+ public VolcanoRuleMatchVisualizer(VolcanoPlanner volcanoPlanner) {
+ this.volcanoPlanner = volcanoPlanner;
+ }
+
+ public void addRuleMatch(String ruleCallID, Collection<? extends RelNode> matchedRels) {
+
+ // store the current state snapshot
+ // nodes contained in the sets
+ // and inputs of relNodes (and relSubsets)
+ Map<String, String> setLabels = new HashMap<>();
+ Map<String, String> setOriginalRel = new HashMap<>();
+ Map<String, Set<String>> nodesInSet = new HashMap<>();
+ Map<String, Set<String>> nodeInputs = new HashMap<>();
+
+ // newNodes appeared after this ruleCall
+ Set<String> newNodes = new HashSet<>();
+
+ // populate current snapshot, and fill in the allNodes map
+ volcanoPlanner.allSets.forEach(set -> {
+ String setID = "set-" + set.id;
+ String setLabel = getSetLabel(set);
+ setLabels.put(setID, setLabel);
+ setOriginalRel.put(setID, set.rel == null ? "" : String.valueOf(set.rel.getId()));
+
+ nodesInSet.put(setID, nodesInSet.getOrDefault(setID, new HashSet<>()));
+
+ Consumer<RelNode> addNode = rel -> {
+ String nodeID = String.valueOf(rel.getId());
+ nodesInSet.get(setID).add(nodeID);
+
+ if (!allNodes.containsKey(nodeID)) {
+ newNodes.add(nodeID);
+ allNodes.put(nodeID, rel);
+ }
+ };
+
+ Consumer<RelNode> addLink = rel -> {
+ String nodeID = String.valueOf(rel.getId());
+ nodeInputs.put(nodeID, new HashSet<>());
+ if (rel instanceof RelSubset) {
+ RelSubset relSubset = (RelSubset) rel;
+ relSubset.getRelList().stream()
+ .filter(input -> input.getTraitSet().equals(relSubset.getTraitSet()))
+ .forEach(input -> nodeInputs.get(nodeID).add(String.valueOf(input.getId())));
Review comment:
This was a design tradeoff to make the links in a set less messy. In the environment I was working on, a set contains multiple subsets with related traits. So a relNode belongs to many subsets at the same time.
For example, suppose there are 2 subsets `S1`, `S2` (`S2` satisfies `S1`) and 2 relnodes `R1`, `R2` (with the same trait as `S2`).
Instead of creating 4 links for relNodes: `R1 -> S1`, `R1 -> S2`, `R2 -> S1`, `R2 -> S2`,
I decided to create 2 links for relNodes: `R1 -> S2` and `R2 -> S2`, and 1 link for the subsets `S2 -> S1`. So the subset satisfaction relationship is indirectly shown.
This way in a set with many subsets/relnodes, the links are much less messy.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@calcite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [calcite] zuozhiw commented on a change in pull request #2486: [CALCITE-4737] Add Volcano Visualizer for Debugging
Posted by GitBox <gi...@apache.org>.
zuozhiw commented on a change in pull request #2486:
URL: https://github.com/apache/calcite/pull/2486#discussion_r689665526
##########
File path: core/src/main/resources/volcano-viz/viz-template.html
##########
@@ -0,0 +1,323 @@
+<!doctype html>
+<html lang="en">
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+<meta charset="utf-8">
+<title>Calcite Rule Match Visualization</title>
+
+<script src="https://d3js.org/d3.v5.min.js" charset="utf-8"></script>
+<script src="https://dagrejs.github.io/project/dagre-d3/latest/dagre-d3.min.js"></script>
+<script src="https://d3js.org/d3-zoom.v1.min.js"></script>
+<script src="https://unpkg.com/tippy.js@3/dist/tippy.all.min.js"></script>
+<script src="volcano-viz-data.js"></script>
+
+<style id="css">
+ body {
+ height: 100%;
+ margin: 0 0;
+ color: #333;
+ font-weight: 300;
+ font-family: "Helvetica Neue", Helvetica, Arial, sans-serf;
+ }
+
+ li a {
+ display: block;
+ /* and you can use padding for additional space if needs, as a clickable area / or other styling */
+ padding: 5px 20px;
+ }
+
+ section {
+ margin-bottom: 3em;
+ }
+
+ section p {
+ text-align: justify;
+ }
+
+ svg {
+ border: 1px solid #ccc;
+ overflow: hidden;
+ margin: 0 auto;
+ }
+
+ pre {
+ border: 1px solid #ccc;
+ }
+
+ .clusters rect {
+ fill: #FFFFE0;
+ stroke: #999;
+ stroke-width: 1.5px;
+ }
+
+ text {
+ font-weight: 300;
+ font-family: "Helvetica Neue", Helvetica, Arial, sans-serf;
+ font-size: 2em;
+ }
+
+ .node rect {
+ stroke: #999;
+ fill: #fff;
+ stroke-width: 1.5px;
+ }
+
+ .edgePath path {
+ stroke: #333;
+ stroke-width: 2px;
+ }
+
+ .container {
+ display: flex;
+ align-items: center;
+ }
+
+ .column1 {
+ flex: 0 0 300px;
+ }
+
+ .column2 {
+ flex: 0 0 1000px;
+ }
+
+ .tippy-content {
+ word-break: break-all;
+ word-wrap: break-word;
+ }
+</style>
+
+<div class="container">
+ <div class="column1">
+ <div style="width: 100%; text-align: center">
+ <div id="current-rule" style="display: block"></div>
+ <button id="prev-button" style="width: 80px; height: 40px; display:inline-block" disabled>prev </button>
+ <button id="next-button" style="width: 80px; height: 40px; display:inline-block" disabled> next</button>
+ </div>
+ <ul id="rule-match-list" style="width: 300px; height: 600px; overflow: auto">
+ </ul>
+ </div>
+ <div class="column2">
+ <svg id="svg-canvas" width="1200px" height="800px"></svg>
+ </div>
+</div>
+
+<script id="js">
+
+ var allNodes = data.allNodes;
+ var ruleMatchSequence = data.ruleMatchSequence;
+ var ruleMatchInfoMap = data.ruleMatchInfoMap;
+ var nodeAddedInRule = data.nodeAddedInRule;
+
+ /*
+ * Graph data and D3 JS render related variables
+ */
+
+ // Create the input graph
+ var g = new dagreD3.graphlib.Graph({
+ compound: true
+ })
+ .setGraph({
+ rankdir: 'LR'
+ })
+ .setDefaultEdgeLabel(function () {
+ return {};
+ });
+
+ // Create the renderer
+ var render = new dagreD3.render();
+
+ // Set up an SVG group so that we can translate the final graph.
+ var svg = d3.select("svg");
+ var svgGroup = svg.append("g");
+
+ // Set up zoom support
+ var svg = d3.select("svg")
+ .attr("width", "1200px")
+ .attr("height", "800px")
+ .call(d3.zoom().on("zoom", function () {
+ svgGroup.attr("transform", d3.event.transform)
+ }));
+
+ /*
+ * Global State
+ */
+
+ var currentRuleID = undefined;
+
+ /*
+ * Event Handler functions
+ */
+
+ var setCurrentRule = (ruleMatchID) => {
+ // un-highlight previous entry
+ var prevRuleID = currentRuleID;
+ if (prevRuleID !== undefined) {
+ var prevRuleElement = document.getElementById(prevRuleID);
+ prevRuleElement.style.backgroundColor = "#FFFFFF";
+ }
+
+ currentRuleID = ruleMatchID;
+ document.getElementById('current-rule').innerText = currentRuleID;
+
+ var currentRuleElement = document.getElementById(currentRuleID);
+ currentRuleElement.style.backgroundColor = "#D3D3D3";
+
+ var ruleIndex = ruleMatchSequence.indexOf(currentRuleID);
+
+ document.getElementById("prev-button").disabled = false;
+ document.getElementById("next-button").disabled = false;
+
+ if (ruleIndex === 0) {
+ document.getElementById("prev-button").disabled = true;
+ }
+ if (ruleIndex === ruleMatchSequence.length - 1) {
+ document.getElementById("next-button").disabled = true;
+ }
+
+ createGraph(ruleMatchID);
+ }
+
+ var createGraph = (ruleMatchID) => {
+ var ruleMatchInfo = ruleMatchInfoMap[ruleMatchID]
+ console.log(ruleMatchInfo);
+
+ // remove previous rendered view and clear graph model
+ d3.select("svg g").selectAll("*").remove();
+ g.nodes().slice().forEach(nodeID => g.removeNode(nodeID));
Review comment:
Agree, but we need to be careful and compare the old nodes with the new nodes. This could be tricky especially when set merge happens: one set is removed and relnodes from this set all go into another set. So re-drawing is eaiser.
One more thing on performance. This tool runs slow when the plan is ultra-large. I investigated and the major performance bottleneck is actually doing the **layout** instead of drawing the nodes. I can't recall the exact numbers but over 90% of the time is spent on the layout (`dagre`) for plans with noticable delay in visualization.
I did an extensive investigation on different layout libraries, but `dagre` remains the best-looking one, so I decided to keep it although it's slow. One sad thing is that none of these layout libraries can calculate layout incrementally. They all have to re-compute the whole thing every time.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@calcite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [calcite] zuozhiw commented on a change in pull request #2486: [CALCITE-4737] Add Volcano Visualizer for Debugging
Posted by GitBox <gi...@apache.org>.
zuozhiw commented on a change in pull request #2486:
URL: https://github.com/apache/calcite/pull/2486#discussion_r689656891
##########
File path: core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRuleMatchVisualizer.java
##########
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.plan.volcano;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptCost;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.tools.visualizer.InputExcludedRelWriter;
+import org.apache.calcite.tools.visualizer.VisualizerNodeInfo;
+import org.apache.calcite.tools.visualizer.VisualizerRuleMatchInfo;
+
+import org.apache.commons.io.IOUtils;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Charsets;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+import static java.util.stream.Collectors.joining;
+
+/**
+ * This is tool to visualize the rule match process of the VolcanoPlanner.
+ *
+ *
+ * <p>To use the visualizer, add a listener before the VolcanoPlanner optimization phase.
+ * Then writes the output to a file after the optimization ends.
+ *
+ * <pre>
+ * // construct the visualizer and attach a listener to VolcanoPlanner
+ * VolcanoRuleMatchVisualizerListener visualizerListener =
+ * new VolcanoRuleMatchVisualizerListener(volcanoPlanner);
+ * volcanoPlanner.addListener(visualizerListener);
+ *
+ * volcanoPlanner.findBestExpr();
+ *
+ * // after the optimization, adds the final best plan
+ * visualizerListener.getVisualizer().addFinalPlan();
+ * // writes the output to files
+ * visualizerListener.getVisualizer().writeToFile(outputDirectory, "");
+ * </pre>
+ */
+public class VolcanoRuleMatchVisualizer {
+
+ VolcanoPlanner volcanoPlanner;
+
+ // a sequence of ruleMatch ID to represent the order of rule match
+ List<String> ruleMatchSequence = new ArrayList<>();
+ // map of ruleMatch ID and the info, including the state snapshot at the time of ruleMatch
+ Map<String, VisualizerRuleMatchInfo> ruleInfoMap = new HashMap<>();
+ // map of nodeID to the ruleID it's first added
+ Map<String, String> nodeAddedInRule = new HashMap<>();
+
+ // a map of relNode ID to the actual RelNode object
+ // contains all the relNodes appear during the optimization
+ // all RelNode are immutable in Calcite, therefore only new nodes will be added
+ Map<String, RelNode> allNodes = new HashMap<>();
+
+ public VolcanoRuleMatchVisualizer(VolcanoPlanner volcanoPlanner) {
+ this.volcanoPlanner = volcanoPlanner;
+ }
+
+ public void addRuleMatch(String ruleCallID, Collection<? extends RelNode> matchedRels) {
+
+ // store the current state snapshot
+ // nodes contained in the sets
+ // and inputs of relNodes (and relSubsets)
+ Map<String, String> setLabels = new HashMap<>();
+ Map<String, String> setOriginalRel = new HashMap<>();
+ Map<String, Set<String>> nodesInSet = new HashMap<>();
+ Map<String, Set<String>> nodeInputs = new HashMap<>();
+
+ // newNodes appeared after this ruleCall
+ Set<String> newNodes = new HashSet<>();
+
+ // populate current snapshot, and fill in the allNodes map
+ volcanoPlanner.allSets.forEach(set -> {
+ String setID = "set-" + set.id;
+ String setLabel = getSetLabel(set);
+ setLabels.put(setID, setLabel);
+ setOriginalRel.put(setID, set.rel == null ? "" : String.valueOf(set.rel.getId()));
+
+ nodesInSet.put(setID, nodesInSet.getOrDefault(setID, new HashSet<>()));
+
+ Consumer<RelNode> addNode = rel -> {
+ String nodeID = String.valueOf(rel.getId());
+ nodesInSet.get(setID).add(nodeID);
+
+ if (!allNodes.containsKey(nodeID)) {
+ newNodes.add(nodeID);
+ allNodes.put(nodeID, rel);
+ }
+ };
+
+ Consumer<RelNode> addLink = rel -> {
+ String nodeID = String.valueOf(rel.getId());
+ nodeInputs.put(nodeID, new HashSet<>());
+ if (rel instanceof RelSubset) {
+ RelSubset relSubset = (RelSubset) rel;
+ relSubset.getRelList().stream()
+ .filter(input -> input.getTraitSet().equals(relSubset.getTraitSet()))
+ .forEach(input -> nodeInputs.get(nodeID).add(String.valueOf(input.getId())));
Review comment:
This was a design tradeoff to make the links in a set less messy. In the environment I was working on, a set contains multiple subsets with related traits. So a relNode belongs to many subsets at the same time.
For example, suppose there are 2 subsets `S1`, `S2` (`S2` satisfies `S1`) and 2 relnodes `R1`, `R2` (with the same trait as `S2`).
Instead of creating 4 links for relNodes: `R1 -> S1`, `R1 -> S2`, `R2 -> S1`, `R2 -> S2`,
I decided to draw 2 links for relNodes: `R1 -> S2` and `R2 -> S2`, and 1 indirect link for the subsets `S2 -> S1`. So the subset satisfaction relationship is indirectly shown.
This way in a set with many subsets/relnodes, the links are much less messy.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@calcite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [calcite] zuozhiw commented on a change in pull request #2486: [CALCITE-4737] Add Volcano Visualizer for Debugging
Posted by GitBox <gi...@apache.org>.
zuozhiw commented on a change in pull request #2486:
URL: https://github.com/apache/calcite/pull/2486#discussion_r689656891
##########
File path: core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRuleMatchVisualizer.java
##########
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.plan.volcano;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptCost;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.tools.visualizer.InputExcludedRelWriter;
+import org.apache.calcite.tools.visualizer.VisualizerNodeInfo;
+import org.apache.calcite.tools.visualizer.VisualizerRuleMatchInfo;
+
+import org.apache.commons.io.IOUtils;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Charsets;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+import static java.util.stream.Collectors.joining;
+
+/**
+ * This is tool to visualize the rule match process of the VolcanoPlanner.
+ *
+ *
+ * <p>To use the visualizer, add a listener before the VolcanoPlanner optimization phase.
+ * Then writes the output to a file after the optimization ends.
+ *
+ * <pre>
+ * // construct the visualizer and attach a listener to VolcanoPlanner
+ * VolcanoRuleMatchVisualizerListener visualizerListener =
+ * new VolcanoRuleMatchVisualizerListener(volcanoPlanner);
+ * volcanoPlanner.addListener(visualizerListener);
+ *
+ * volcanoPlanner.findBestExpr();
+ *
+ * // after the optimization, adds the final best plan
+ * visualizerListener.getVisualizer().addFinalPlan();
+ * // writes the output to files
+ * visualizerListener.getVisualizer().writeToFile(outputDirectory, "");
+ * </pre>
+ */
+public class VolcanoRuleMatchVisualizer {
+
+ VolcanoPlanner volcanoPlanner;
+
+ // a sequence of ruleMatch ID to represent the order of rule match
+ List<String> ruleMatchSequence = new ArrayList<>();
+ // map of ruleMatch ID and the info, including the state snapshot at the time of ruleMatch
+ Map<String, VisualizerRuleMatchInfo> ruleInfoMap = new HashMap<>();
+ // map of nodeID to the ruleID it's first added
+ Map<String, String> nodeAddedInRule = new HashMap<>();
+
+ // a map of relNode ID to the actual RelNode object
+ // contains all the relNodes appear during the optimization
+ // all RelNode are immutable in Calcite, therefore only new nodes will be added
+ Map<String, RelNode> allNodes = new HashMap<>();
+
+ public VolcanoRuleMatchVisualizer(VolcanoPlanner volcanoPlanner) {
+ this.volcanoPlanner = volcanoPlanner;
+ }
+
+ public void addRuleMatch(String ruleCallID, Collection<? extends RelNode> matchedRels) {
+
+ // store the current state snapshot
+ // nodes contained in the sets
+ // and inputs of relNodes (and relSubsets)
+ Map<String, String> setLabels = new HashMap<>();
+ Map<String, String> setOriginalRel = new HashMap<>();
+ Map<String, Set<String>> nodesInSet = new HashMap<>();
+ Map<String, Set<String>> nodeInputs = new HashMap<>();
+
+ // newNodes appeared after this ruleCall
+ Set<String> newNodes = new HashSet<>();
+
+ // populate current snapshot, and fill in the allNodes map
+ volcanoPlanner.allSets.forEach(set -> {
+ String setID = "set-" + set.id;
+ String setLabel = getSetLabel(set);
+ setLabels.put(setID, setLabel);
+ setOriginalRel.put(setID, set.rel == null ? "" : String.valueOf(set.rel.getId()));
+
+ nodesInSet.put(setID, nodesInSet.getOrDefault(setID, new HashSet<>()));
+
+ Consumer<RelNode> addNode = rel -> {
+ String nodeID = String.valueOf(rel.getId());
+ nodesInSet.get(setID).add(nodeID);
+
+ if (!allNodes.containsKey(nodeID)) {
+ newNodes.add(nodeID);
+ allNodes.put(nodeID, rel);
+ }
+ };
+
+ Consumer<RelNode> addLink = rel -> {
+ String nodeID = String.valueOf(rel.getId());
+ nodeInputs.put(nodeID, new HashSet<>());
+ if (rel instanceof RelSubset) {
+ RelSubset relSubset = (RelSubset) rel;
+ relSubset.getRelList().stream()
+ .filter(input -> input.getTraitSet().equals(relSubset.getTraitSet()))
+ .forEach(input -> nodeInputs.get(nodeID).add(String.valueOf(input.getId())));
Review comment:
This was a design tradeoff to make the links in a set less messy. In the environment I was working on, a set contains multiple subsets with related traits. So a relNode belongs to many subsets at the same time.
For example, supposethere are 2 subsets `S1`, `S2` (`S2` satisfies `S1`) and 2 relnodes `R1`, `R2` with the same trait as `S2`.
Instead of creating 4 links for relNodes: `R1 -> S1`, `R1 -> S2`, `R2 -> S1`, `R2 -> S2`,
I decided to draw 2 links for relNodes: `R1 -> S2` and `R2 -> S2`, and 1 indirect link for the subsets `S2 -> S1`. So the subset satisfaction relationship is indirectly shown.
This way in a set with many subsets/relnodes, the links are much less messy.
##########
File path: core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRuleMatchVisualizer.java
##########
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.plan.volcano;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptCost;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.tools.visualizer.InputExcludedRelWriter;
+import org.apache.calcite.tools.visualizer.VisualizerNodeInfo;
+import org.apache.calcite.tools.visualizer.VisualizerRuleMatchInfo;
+
+import org.apache.commons.io.IOUtils;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Charsets;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+import static java.util.stream.Collectors.joining;
+
+/**
+ * This is tool to visualize the rule match process of the VolcanoPlanner.
+ *
+ *
+ * <p>To use the visualizer, add a listener before the VolcanoPlanner optimization phase.
+ * Then writes the output to a file after the optimization ends.
+ *
+ * <pre>
+ * // construct the visualizer and attach a listener to VolcanoPlanner
+ * VolcanoRuleMatchVisualizerListener visualizerListener =
+ * new VolcanoRuleMatchVisualizerListener(volcanoPlanner);
+ * volcanoPlanner.addListener(visualizerListener);
+ *
+ * volcanoPlanner.findBestExpr();
+ *
+ * // after the optimization, adds the final best plan
+ * visualizerListener.getVisualizer().addFinalPlan();
+ * // writes the output to files
+ * visualizerListener.getVisualizer().writeToFile(outputDirectory, "");
+ * </pre>
+ */
+public class VolcanoRuleMatchVisualizer {
+
+ VolcanoPlanner volcanoPlanner;
+
+ // a sequence of ruleMatch ID to represent the order of rule match
+ List<String> ruleMatchSequence = new ArrayList<>();
+ // map of ruleMatch ID and the info, including the state snapshot at the time of ruleMatch
+ Map<String, VisualizerRuleMatchInfo> ruleInfoMap = new HashMap<>();
+ // map of nodeID to the ruleID it's first added
+ Map<String, String> nodeAddedInRule = new HashMap<>();
+
+ // a map of relNode ID to the actual RelNode object
+ // contains all the relNodes appear during the optimization
+ // all RelNode are immutable in Calcite, therefore only new nodes will be added
+ Map<String, RelNode> allNodes = new HashMap<>();
+
+ public VolcanoRuleMatchVisualizer(VolcanoPlanner volcanoPlanner) {
+ this.volcanoPlanner = volcanoPlanner;
+ }
+
+ public void addRuleMatch(String ruleCallID, Collection<? extends RelNode> matchedRels) {
+
+ // store the current state snapshot
+ // nodes contained in the sets
+ // and inputs of relNodes (and relSubsets)
+ Map<String, String> setLabels = new HashMap<>();
+ Map<String, String> setOriginalRel = new HashMap<>();
+ Map<String, Set<String>> nodesInSet = new HashMap<>();
+ Map<String, Set<String>> nodeInputs = new HashMap<>();
+
+ // newNodes appeared after this ruleCall
+ Set<String> newNodes = new HashSet<>();
+
+ // populate current snapshot, and fill in the allNodes map
+ volcanoPlanner.allSets.forEach(set -> {
+ String setID = "set-" + set.id;
+ String setLabel = getSetLabel(set);
+ setLabels.put(setID, setLabel);
+ setOriginalRel.put(setID, set.rel == null ? "" : String.valueOf(set.rel.getId()));
+
+ nodesInSet.put(setID, nodesInSet.getOrDefault(setID, new HashSet<>()));
+
+ Consumer<RelNode> addNode = rel -> {
+ String nodeID = String.valueOf(rel.getId());
+ nodesInSet.get(setID).add(nodeID);
+
+ if (!allNodes.containsKey(nodeID)) {
+ newNodes.add(nodeID);
+ allNodes.put(nodeID, rel);
+ }
+ };
+
+ Consumer<RelNode> addLink = rel -> {
+ String nodeID = String.valueOf(rel.getId());
+ nodeInputs.put(nodeID, new HashSet<>());
+ if (rel instanceof RelSubset) {
+ RelSubset relSubset = (RelSubset) rel;
+ relSubset.getRelList().stream()
+ .filter(input -> input.getTraitSet().equals(relSubset.getTraitSet()))
+ .forEach(input -> nodeInputs.get(nodeID).add(String.valueOf(input.getId())));
Review comment:
This was a design tradeoff to make the links in a set less messy. In the environment I was working on, a set contains multiple subsets with related traits. So a relNode belongs to many subsets at the same time.
For example, suppose there are 2 subsets `S1`, `S2` (`S2` satisfies `S1`) and 2 relnodes `R1`, `R2` with the same trait as `S2`.
Instead of creating 4 links for relNodes: `R1 -> S1`, `R1 -> S2`, `R2 -> S1`, `R2 -> S2`,
I decided to draw 2 links for relNodes: `R1 -> S2` and `R2 -> S2`, and 1 indirect link for the subsets `S2 -> S1`. So the subset satisfaction relationship is indirectly shown.
This way in a set with many subsets/relnodes, the links are much less messy.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@calcite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [calcite] thomasrebele commented on a change in pull request #2486: [CALCITE-4737] Add Volcano Visualizer for Debugging
Posted by GitBox <gi...@apache.org>.
thomasrebele commented on a change in pull request #2486:
URL: https://github.com/apache/calcite/pull/2486#discussion_r689428607
##########
File path: core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRuleMatchVisualizer.java
##########
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.plan.volcano;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptCost;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.tools.visualizer.InputExcludedRelWriter;
+import org.apache.calcite.tools.visualizer.VisualizerNodeInfo;
+import org.apache.calcite.tools.visualizer.VisualizerRuleMatchInfo;
+
+import org.apache.commons.io.IOUtils;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Charsets;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+import static java.util.stream.Collectors.joining;
+
+/**
+ * This is tool to visualize the rule match process of the VolcanoPlanner.
+ *
+ *
+ * <p>To use the visualizer, add a listener before the VolcanoPlanner optimization phase.
+ * Then writes the output to a file after the optimization ends.
+ *
+ * <pre>
+ * // construct the visualizer and attach a listener to VolcanoPlanner
+ * VolcanoRuleMatchVisualizerListener visualizerListener =
+ * new VolcanoRuleMatchVisualizerListener(volcanoPlanner);
+ * volcanoPlanner.addListener(visualizerListener);
+ *
+ * volcanoPlanner.findBestExpr();
+ *
+ * // after the optimization, adds the final best plan
+ * visualizerListener.getVisualizer().addFinalPlan();
+ * // writes the output to files
+ * visualizerListener.getVisualizer().writeToFile(outputDirectory, "");
+ * </pre>
+ */
+public class VolcanoRuleMatchVisualizer {
+
+ VolcanoPlanner volcanoPlanner;
+
+ // a sequence of ruleMatch ID to represent the order of rule match
+ List<String> ruleMatchSequence = new ArrayList<>();
+ // map of ruleMatch ID and the info, including the state snapshot at the time of ruleMatch
+ Map<String, VisualizerRuleMatchInfo> ruleInfoMap = new HashMap<>();
+ // map of nodeID to the ruleID it's first added
+ Map<String, String> nodeAddedInRule = new HashMap<>();
+
+ // a map of relNode ID to the actual RelNode object
+ // contains all the relNodes appear during the optimization
+ // all RelNode are immutable in Calcite, therefore only new nodes will be added
Review comment:
The children of the RelNodes can be modified by RelNode#replaceInput, so I would prefer to rephrase it. Maybe we just remove this line?
##########
File path: core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRuleMatchVisualizer.java
##########
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.plan.volcano;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptCost;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.tools.visualizer.InputExcludedRelWriter;
+import org.apache.calcite.tools.visualizer.VisualizerNodeInfo;
+import org.apache.calcite.tools.visualizer.VisualizerRuleMatchInfo;
+
+import org.apache.commons.io.IOUtils;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Charsets;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+import static java.util.stream.Collectors.joining;
+
+/**
+ * This is tool to visualize the rule match process of the VolcanoPlanner.
+ *
+ *
+ * <p>To use the visualizer, add a listener before the VolcanoPlanner optimization phase.
+ * Then writes the output to a file after the optimization ends.
+ *
+ * <pre>
+ * // construct the visualizer and attach a listener to VolcanoPlanner
+ * VolcanoRuleMatchVisualizerListener visualizerListener =
+ * new VolcanoRuleMatchVisualizerListener(volcanoPlanner);
+ * volcanoPlanner.addListener(visualizerListener);
+ *
+ * volcanoPlanner.findBestExpr();
+ *
+ * // after the optimization, adds the final best plan
+ * visualizerListener.getVisualizer().addFinalPlan();
+ * // writes the output to files
+ * visualizerListener.getVisualizer().writeToFile(outputDirectory, "");
+ * </pre>
+ */
+public class VolcanoRuleMatchVisualizer {
+
+ VolcanoPlanner volcanoPlanner;
+
+ // a sequence of ruleMatch ID to represent the order of rule match
+ List<String> ruleMatchSequence = new ArrayList<>();
+ // map of ruleMatch ID and the info, including the state snapshot at the time of ruleMatch
+ Map<String, VisualizerRuleMatchInfo> ruleInfoMap = new HashMap<>();
+ // map of nodeID to the ruleID it's first added
+ Map<String, String> nodeAddedInRule = new HashMap<>();
+
+ // a map of relNode ID to the actual RelNode object
+ // contains all the relNodes appear during the optimization
+ // all RelNode are immutable in Calcite, therefore only new nodes will be added
+ Map<String, RelNode> allNodes = new HashMap<>();
+
+ public VolcanoRuleMatchVisualizer(VolcanoPlanner volcanoPlanner) {
+ this.volcanoPlanner = volcanoPlanner;
+ }
+
+ public void addRuleMatch(String ruleCallID, Collection<? extends RelNode> matchedRels) {
+
+ // store the current state snapshot
+ // nodes contained in the sets
+ // and inputs of relNodes (and relSubsets)
+ Map<String, String> setLabels = new HashMap<>();
+ Map<String, String> setOriginalRel = new HashMap<>();
+ Map<String, Set<String>> nodesInSet = new HashMap<>();
+ Map<String, Set<String>> nodeInputs = new HashMap<>();
+
+ // newNodes appeared after this ruleCall
+ Set<String> newNodes = new HashSet<>();
+
+ // populate current snapshot, and fill in the allNodes map
+ volcanoPlanner.allSets.forEach(set -> {
+ String setID = "set-" + set.id;
+ String setLabel = getSetLabel(set);
+ setLabels.put(setID, setLabel);
+ setOriginalRel.put(setID, set.rel == null ? "" : String.valueOf(set.rel.getId()));
+
+ nodesInSet.put(setID, nodesInSet.getOrDefault(setID, new HashSet<>()));
+
+ Consumer<RelNode> addNode = rel -> {
+ String nodeID = String.valueOf(rel.getId());
+ nodesInSet.get(setID).add(nodeID);
+
+ if (!allNodes.containsKey(nodeID)) {
+ newNodes.add(nodeID);
+ allNodes.put(nodeID, rel);
+ }
+ };
+
+ Consumer<RelNode> addLink = rel -> {
+ String nodeID = String.valueOf(rel.getId());
+ nodeInputs.put(nodeID, new HashSet<>());
Review comment:
Assigning the new HashSet to a variable and using it below instead of nodeinputs.get(nodeId) would save a lot of lookups.
##########
File path: core/src/main/resources/volcano-viz/viz-template.html
##########
@@ -0,0 +1,323 @@
+<!doctype html>
+<html lang="en">
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+<meta charset="utf-8">
+<title>Calcite Rule Match Visualization</title>
+
+<script src="https://d3js.org/d3.v5.min.js" charset="utf-8"></script>
+<script src="https://dagrejs.github.io/project/dagre-d3/latest/dagre-d3.min.js"></script>
+<script src="https://d3js.org/d3-zoom.v1.min.js"></script>
+<script src="https://unpkg.com/tippy.js@3/dist/tippy.all.min.js"></script>
+<script src="volcano-viz-data.js"></script>
+
+<style id="css">
+ body {
+ height: 100%;
+ margin: 0 0;
+ color: #333;
+ font-weight: 300;
+ font-family: "Helvetica Neue", Helvetica, Arial, sans-serf;
+ }
+
+ li a {
+ display: block;
+ /* and you can use padding for additional space if needs, as a clickable area / or other styling */
+ padding: 5px 20px;
+ }
+
+ section {
+ margin-bottom: 3em;
+ }
+
+ section p {
+ text-align: justify;
+ }
+
+ svg {
+ border: 1px solid #ccc;
+ overflow: hidden;
+ margin: 0 auto;
+ }
+
+ pre {
+ border: 1px solid #ccc;
+ }
+
+ .clusters rect {
+ fill: #FFFFE0;
+ stroke: #999;
+ stroke-width: 1.5px;
+ }
+
+ text {
+ font-weight: 300;
+ font-family: "Helvetica Neue", Helvetica, Arial, sans-serf;
+ font-size: 2em;
+ }
+
+ .node rect {
+ stroke: #999;
+ fill: #fff;
+ stroke-width: 1.5px;
+ }
+
+ .edgePath path {
+ stroke: #333;
+ stroke-width: 2px;
+ }
+
+ .container {
+ display: flex;
+ align-items: center;
+ }
+
+ .column1 {
+ flex: 0 0 300px;
+ }
+
+ .column2 {
+ flex: 0 0 1000px;
+ }
+
+ .tippy-content {
+ word-break: break-all;
+ word-wrap: break-word;
+ }
+</style>
+
+<div class="container">
+ <div class="column1">
+ <div style="width: 100%; text-align: center">
+ <div id="current-rule" style="display: block"></div>
+ <button id="prev-button" style="width: 80px; height: 40px; display:inline-block" disabled>prev </button>
+ <button id="next-button" style="width: 80px; height: 40px; display:inline-block" disabled> next</button>
+ </div>
+ <ul id="rule-match-list" style="width: 300px; height: 600px; overflow: auto">
+ </ul>
+ </div>
+ <div class="column2">
+ <svg id="svg-canvas" width="1200px" height="800px"></svg>
+ </div>
+</div>
+
+<script id="js">
+
+ var allNodes = data.allNodes;
+ var ruleMatchSequence = data.ruleMatchSequence;
+ var ruleMatchInfoMap = data.ruleMatchInfoMap;
+ var nodeAddedInRule = data.nodeAddedInRule;
+
+ /*
+ * Graph data and D3 JS render related variables
+ */
+
+ // Create the input graph
+ var g = new dagreD3.graphlib.Graph({
+ compound: true
+ })
+ .setGraph({
+ rankdir: 'LR'
+ })
+ .setDefaultEdgeLabel(function () {
+ return {};
+ });
+
+ // Create the renderer
+ var render = new dagreD3.render();
+
+ // Set up an SVG group so that we can translate the final graph.
+ var svg = d3.select("svg");
+ var svgGroup = svg.append("g");
+
+ // Set up zoom support
+ var svg = d3.select("svg")
+ .attr("width", "1200px")
+ .attr("height", "800px")
+ .call(d3.zoom().on("zoom", function () {
+ svgGroup.attr("transform", d3.event.transform)
+ }));
+
+ /*
+ * Global State
+ */
+
+ var currentRuleID = undefined;
+
+ /*
+ * Event Handler functions
+ */
+
+ var setCurrentRule = (ruleMatchID) => {
+ // un-highlight previous entry
+ var prevRuleID = currentRuleID;
+ if (prevRuleID !== undefined) {
+ var prevRuleElement = document.getElementById(prevRuleID);
+ prevRuleElement.style.backgroundColor = "#FFFFFF";
+ }
+
+ currentRuleID = ruleMatchID;
+ document.getElementById('current-rule').innerText = currentRuleID;
+
+ var currentRuleElement = document.getElementById(currentRuleID);
+ currentRuleElement.style.backgroundColor = "#D3D3D3";
+
+ var ruleIndex = ruleMatchSequence.indexOf(currentRuleID);
+
+ document.getElementById("prev-button").disabled = false;
+ document.getElementById("next-button").disabled = false;
+
+ if (ruleIndex === 0) {
+ document.getElementById("prev-button").disabled = true;
+ }
+ if (ruleIndex === ruleMatchSequence.length - 1) {
+ document.getElementById("next-button").disabled = true;
+ }
+
+ createGraph(ruleMatchID);
+ }
+
+ var createGraph = (ruleMatchID) => {
+ var ruleMatchInfo = ruleMatchInfoMap[ruleMatchID]
+ console.log(ruleMatchInfo);
+
+ // remove previous rendered view and clear graph model
+ d3.select("svg g").selectAll("*").remove();
+ g.nodes().slice().forEach(nodeID => g.removeNode(nodeID));
Review comment:
I think it would be better to update the graph instead of resetting it. We can leave it like this for the moment and change it when we add a functionality that would benefit from keeping the existing nodes.
##########
File path: core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRuleMatchVisualizer.java
##########
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.plan.volcano;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptCost;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.tools.visualizer.InputExcludedRelWriter;
+import org.apache.calcite.tools.visualizer.VisualizerNodeInfo;
+import org.apache.calcite.tools.visualizer.VisualizerRuleMatchInfo;
+
+import org.apache.commons.io.IOUtils;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Charsets;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+import static java.util.stream.Collectors.joining;
+
+/**
+ * This is tool to visualize the rule match process of the VolcanoPlanner.
+ *
+ *
+ * <p>To use the visualizer, add a listener before the VolcanoPlanner optimization phase.
+ * Then writes the output to a file after the optimization ends.
+ *
+ * <pre>
+ * // construct the visualizer and attach a listener to VolcanoPlanner
+ * VolcanoRuleMatchVisualizerListener visualizerListener =
+ * new VolcanoRuleMatchVisualizerListener(volcanoPlanner);
+ * volcanoPlanner.addListener(visualizerListener);
+ *
+ * volcanoPlanner.findBestExpr();
+ *
+ * // after the optimization, adds the final best plan
+ * visualizerListener.getVisualizer().addFinalPlan();
+ * // writes the output to files
+ * visualizerListener.getVisualizer().writeToFile(outputDirectory, "");
+ * </pre>
+ */
+public class VolcanoRuleMatchVisualizer {
+
+ VolcanoPlanner volcanoPlanner;
+
+ // a sequence of ruleMatch ID to represent the order of rule match
+ List<String> ruleMatchSequence = new ArrayList<>();
+ // map of ruleMatch ID and the info, including the state snapshot at the time of ruleMatch
+ Map<String, VisualizerRuleMatchInfo> ruleInfoMap = new HashMap<>();
+ // map of nodeID to the ruleID it's first added
+ Map<String, String> nodeAddedInRule = new HashMap<>();
+
+ // a map of relNode ID to the actual RelNode object
+ // contains all the relNodes appear during the optimization
+ // all RelNode are immutable in Calcite, therefore only new nodes will be added
+ Map<String, RelNode> allNodes = new HashMap<>();
+
+ public VolcanoRuleMatchVisualizer(VolcanoPlanner volcanoPlanner) {
+ this.volcanoPlanner = volcanoPlanner;
+ }
+
+ public void addRuleMatch(String ruleCallID, Collection<? extends RelNode> matchedRels) {
+
+ // store the current state snapshot
+ // nodes contained in the sets
+ // and inputs of relNodes (and relSubsets)
+ Map<String, String> setLabels = new HashMap<>();
+ Map<String, String> setOriginalRel = new HashMap<>();
+ Map<String, Set<String>> nodesInSet = new HashMap<>();
+ Map<String, Set<String>> nodeInputs = new HashMap<>();
+
+ // newNodes appeared after this ruleCall
+ Set<String> newNodes = new HashSet<>();
+
+ // populate current snapshot, and fill in the allNodes map
+ volcanoPlanner.allSets.forEach(set -> {
+ String setID = "set-" + set.id;
+ String setLabel = getSetLabel(set);
+ setLabels.put(setID, setLabel);
+ setOriginalRel.put(setID, set.rel == null ? "" : String.valueOf(set.rel.getId()));
+
+ nodesInSet.put(setID, nodesInSet.getOrDefault(setID, new HashSet<>()));
+
+ Consumer<RelNode> addNode = rel -> {
+ String nodeID = String.valueOf(rel.getId());
+ nodesInSet.get(setID).add(nodeID);
+
+ if (!allNodes.containsKey(nodeID)) {
+ newNodes.add(nodeID);
+ allNodes.put(nodeID, rel);
+ }
+ };
+
+ Consumer<RelNode> addLink = rel -> {
+ String nodeID = String.valueOf(rel.getId());
+ nodeInputs.put(nodeID, new HashSet<>());
+ if (rel instanceof RelSubset) {
+ RelSubset relSubset = (RelSubset) rel;
+ relSubset.getRelList().stream()
+ .filter(input -> input.getTraitSet().equals(relSubset.getTraitSet()))
+ .forEach(input -> nodeInputs.get(nodeID).add(String.valueOf(input.getId())));
+ relSubset.set.subsets.stream()
+ .filter(other -> !other.equals(relSubset))
+ .filter(other -> other.getTraitSet().satisfies(relSubset.getTraitSet()))
+ .forEach(other -> nodeInputs.get(nodeID).add(String.valueOf(other.getId())));
+ } else {
+ rel.getInputs().forEach(input -> nodeInputs.get(nodeID)
+ .add(String.valueOf(input.getId())));
+ }
+ };
+
+ set.rels.forEach(addNode);
+ set.subsets.forEach(addNode);
+ set.rels.forEach(addLink);
+ set.subsets.forEach(addLink);
+ });
+
+ // get the matched nodes of this rule
+ Set<String> matchedNodeIDs = matchedRels.stream()
+ .map(rel -> String.valueOf(rel.getId()))
+ .collect(Collectors.toSet());
+
+ // get importance 0 rels as of right now
+ Set<String> importanceZeroNodes = new HashSet<>();
Review comment:
Maybe it's better to keep the term `prunedNodes` used by the VolcanoPlanner?
##########
File path: core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRuleMatchVisualizer.java
##########
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.plan.volcano;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptCost;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.tools.visualizer.InputExcludedRelWriter;
+import org.apache.calcite.tools.visualizer.VisualizerNodeInfo;
+import org.apache.calcite.tools.visualizer.VisualizerRuleMatchInfo;
+
+import org.apache.commons.io.IOUtils;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Charsets;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+import static java.util.stream.Collectors.joining;
+
+/**
+ * This is tool to visualize the rule match process of the VolcanoPlanner.
+ *
+ *
+ * <p>To use the visualizer, add a listener before the VolcanoPlanner optimization phase.
+ * Then writes the output to a file after the optimization ends.
+ *
+ * <pre>
+ * // construct the visualizer and attach a listener to VolcanoPlanner
+ * VolcanoRuleMatchVisualizerListener visualizerListener =
+ * new VolcanoRuleMatchVisualizerListener(volcanoPlanner);
+ * volcanoPlanner.addListener(visualizerListener);
Review comment:
how about adding the listener when constructing the visualizer?
##########
File path: core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRuleMatchVisualizer.java
##########
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.plan.volcano;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptCost;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.tools.visualizer.InputExcludedRelWriter;
+import org.apache.calcite.tools.visualizer.VisualizerNodeInfo;
+import org.apache.calcite.tools.visualizer.VisualizerRuleMatchInfo;
+
+import org.apache.commons.io.IOUtils;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Charsets;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+import static java.util.stream.Collectors.joining;
+
+/**
+ * This is tool to visualize the rule match process of the VolcanoPlanner.
+ *
+ *
+ * <p>To use the visualizer, add a listener before the VolcanoPlanner optimization phase.
+ * Then writes the output to a file after the optimization ends.
+ *
+ * <pre>
+ * // construct the visualizer and attach a listener to VolcanoPlanner
+ * VolcanoRuleMatchVisualizerListener visualizerListener =
+ * new VolcanoRuleMatchVisualizerListener(volcanoPlanner);
+ * volcanoPlanner.addListener(visualizerListener);
+ *
+ * volcanoPlanner.findBestExpr();
+ *
+ * // after the optimization, adds the final best plan
+ * visualizerListener.getVisualizer().addFinalPlan();
+ * // writes the output to files
+ * visualizerListener.getVisualizer().writeToFile(outputDirectory, "");
+ * </pre>
+ */
+public class VolcanoRuleMatchVisualizer {
+
+ VolcanoPlanner volcanoPlanner;
+
+ // a sequence of ruleMatch ID to represent the order of rule match
+ List<String> ruleMatchSequence = new ArrayList<>();
+ // map of ruleMatch ID and the info, including the state snapshot at the time of ruleMatch
+ Map<String, VisualizerRuleMatchInfo> ruleInfoMap = new HashMap<>();
+ // map of nodeID to the ruleID it's first added
+ Map<String, String> nodeAddedInRule = new HashMap<>();
+
+ // a map of relNode ID to the actual RelNode object
+ // contains all the relNodes appear during the optimization
+ // all RelNode are immutable in Calcite, therefore only new nodes will be added
+ Map<String, RelNode> allNodes = new HashMap<>();
+
+ public VolcanoRuleMatchVisualizer(VolcanoPlanner volcanoPlanner) {
+ this.volcanoPlanner = volcanoPlanner;
+ }
+
+ public void addRuleMatch(String ruleCallID, Collection<? extends RelNode> matchedRels) {
+
+ // store the current state snapshot
+ // nodes contained in the sets
+ // and inputs of relNodes (and relSubsets)
+ Map<String, String> setLabels = new HashMap<>();
+ Map<String, String> setOriginalRel = new HashMap<>();
+ Map<String, Set<String>> nodesInSet = new HashMap<>();
+ Map<String, Set<String>> nodeInputs = new HashMap<>();
+
+ // newNodes appeared after this ruleCall
+ Set<String> newNodes = new HashSet<>();
+
+ // populate current snapshot, and fill in the allNodes map
+ volcanoPlanner.allSets.forEach(set -> {
+ String setID = "set-" + set.id;
+ String setLabel = getSetLabel(set);
+ setLabels.put(setID, setLabel);
+ setOriginalRel.put(setID, set.rel == null ? "" : String.valueOf(set.rel.getId()));
+
+ nodesInSet.put(setID, nodesInSet.getOrDefault(setID, new HashSet<>()));
+
+ Consumer<RelNode> addNode = rel -> {
+ String nodeID = String.valueOf(rel.getId());
+ nodesInSet.get(setID).add(nodeID);
+
+ if (!allNodes.containsKey(nodeID)) {
+ newNodes.add(nodeID);
+ allNodes.put(nodeID, rel);
+ }
+ };
+
+ Consumer<RelNode> addLink = rel -> {
+ String nodeID = String.valueOf(rel.getId());
+ nodeInputs.put(nodeID, new HashSet<>());
+ if (rel instanceof RelSubset) {
+ RelSubset relSubset = (RelSubset) rel;
+ relSubset.getRelList().stream()
+ .filter(input -> input.getTraitSet().equals(relSubset.getTraitSet()))
+ .forEach(input -> nodeInputs.get(nodeID).add(String.valueOf(input.getId())));
+ relSubset.set.subsets.stream()
+ .filter(other -> !other.equals(relSubset))
+ .filter(other -> other.getTraitSet().satisfies(relSubset.getTraitSet()))
+ .forEach(other -> nodeInputs.get(nodeID).add(String.valueOf(other.getId())));
+ } else {
+ rel.getInputs().forEach(input -> nodeInputs.get(nodeID)
+ .add(String.valueOf(input.getId())));
+ }
+ };
+
+ set.rels.forEach(addNode);
+ set.subsets.forEach(addNode);
+ set.rels.forEach(addLink);
+ set.subsets.forEach(addLink);
+ });
+
+ // get the matched nodes of this rule
+ Set<String> matchedNodeIDs = matchedRels.stream()
+ .map(rel -> String.valueOf(rel.getId()))
+ .collect(Collectors.toSet());
+
+ // get importance 0 rels as of right now
+ Set<String> importanceZeroNodes = new HashSet<>();
+ volcanoPlanner.prunedNodes
+ .forEach(rel -> importanceZeroNodes.add(Integer.toString(rel.getId())));
+
+ VisualizerRuleMatchInfo ruleMatchInfo =
+ new VisualizerRuleMatchInfo(setLabels, setOriginalRel, nodesInSet,
+ nodeInputs, matchedNodeIDs, newNodes, importanceZeroNodes);
+
+ ruleMatchSequence.add(ruleCallID);
+ ruleInfoMap.put(ruleCallID, ruleMatchInfo);
+
+ newNodes.forEach(newNode -> nodeAddedInRule.put(newNode, ruleCallID));
+ }
+
+ /**
+ * Add a final plan to the variable.
+ */
+ public void addFinalPlan() {
+ assert !ruleMatchSequence.contains("FINAL");
+
+ Set<RelNode> finalPlanNodes = new HashSet<>();
+ Deque<RelSubset> subsetsToVisit = new LinkedList<>();
+ subsetsToVisit.add((RelSubset) volcanoPlanner.getRoot());
+
+ RelSubset subset;
+ while ((subset = subsetsToVisit.poll()) != null) {
+ // add subset itself to the highlight list
+ finalPlanNodes.add(subset);
+ // highlight its best node if it exists
+ RelNode best = subset.getBest();
+ if (best == null) {
+ continue;
+ }
+ finalPlanNodes.add(best);
+ // recursively visit the input relSubsets of the best node
+ best.getInputs().stream().map(rel -> (RelSubset) rel).forEach(subsetsToVisit::add);
+ }
+
+ this.addRuleMatch("FINAL", new ArrayList<>(finalPlanNodes));
+ }
+
+ private String getSetLabel(RelSet set) {
+ return "set-" + set.id + " ";
+ }
+
+ private String getJsonStringResult() {
+ try {
+ Map<String, VisualizerNodeInfo> nodeInfoMap = new HashMap<>();
+ for (String nodeID : allNodes.keySet()) {
+ RelNode relNode = allNodes.get(nodeID);
+ RelNode root = volcanoPlanner.getRoot();
+ if (root == null) {
+ throw new RuntimeException("volcano planner root is null");
+ }
+ RelOptCluster cluster = root.getCluster();
+ RelOptCost cost = volcanoPlanner.getCost(relNode, cluster.getMetadataQuery());
+ Double rowCount =
+ relNode.getCluster().getMetadataQuery().getRowCount(relNode);
+
+ VisualizerNodeInfo nodeInfo;
+ if (relNode instanceof RelSubset) {
+ RelSubset relSubset = (RelSubset) relNode;
+ String nodeLabel = "subset#" + relSubset.getId() + "-set#" + relSubset.set.id + "-\n"
+ + relSubset.getTraitSet().toString();
+ String relIDs = relSubset.getRelList().stream()
+ .map(i -> "#" + i.getId()).collect(joining(", "));
+ String explanation = "rels: [" + relIDs + "]";
+ nodeInfo =
+ new VisualizerNodeInfo(nodeLabel, true, explanation, cost, rowCount);
+ } else {
+ InputExcludedRelWriter relWriter = new InputExcludedRelWriter();
+ relNode.explain(relWriter);
+ String inputIDs = relNode.getInputs().stream()
+ .map(i -> "#" + i.getId()).collect(joining(", "));
+ String explanation = relWriter.toString() + ", inputs: [" + inputIDs + "]";
+
+ String nodeLabel = "#" + relNode.getId() + "-" + relNode.getRelTypeName();
+ nodeInfo = new VisualizerNodeInfo(nodeLabel, false, explanation, cost,
+ rowCount);
+ }
+
+ nodeInfoMap.put(nodeID, nodeInfo);
+ }
+
+ HashMap<String, Object> data = new HashMap<>();
+ data.put("allNodes", nodeInfoMap);
+ data.put("ruleMatchSequence", ruleMatchSequence);
+ data.put("ruleMatchInfoMap", ruleInfoMap);
+ data.put("nodeAddedInRule", nodeAddedInRule);
+
+ ObjectMapper objectMapper = new ObjectMapper();
+ return objectMapper.writeValueAsString(data);
+ } catch (JsonProcessingException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Writes the HTML and JS files of the rule match visualization.
+ * <p>
+ * The old files with the same name will be replaced.
+ *
+ * @param outputDirectory directory of the output files
+ * @param suffix file name suffix, can be null
+ */
+ public void writeToFile(String outputDirectory, String suffix) {
+ // default HTML template is under "resources"
+ writeToFile("volcano-viz", outputDirectory, suffix);
+ }
+
+ public void writeToFile(String templateDirectory, String outputDirectory, String suffix) {
+ try {
+ String templatePath = Paths.get(templateDirectory).resolve("viz-template.html").toString();
+ String htmlTemplate = IOUtils.toString(getClass().getResourceAsStream(templatePath),
Review comment:
Calling getClass().getResourceAsStream(...) tries to access a wrong path, while getClass().getClassLoader().getResourceAsStream() seems to work.
##########
File path: core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRuleMatchVisualizer.java
##########
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.plan.volcano;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptCost;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.tools.visualizer.InputExcludedRelWriter;
+import org.apache.calcite.tools.visualizer.VisualizerNodeInfo;
+import org.apache.calcite.tools.visualizer.VisualizerRuleMatchInfo;
+
+import org.apache.commons.io.IOUtils;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Charsets;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+import static java.util.stream.Collectors.joining;
+
+/**
+ * This is tool to visualize the rule match process of the VolcanoPlanner.
+ *
+ *
+ * <p>To use the visualizer, add a listener before the VolcanoPlanner optimization phase.
+ * Then writes the output to a file after the optimization ends.
+ *
+ * <pre>
+ * // construct the visualizer and attach a listener to VolcanoPlanner
+ * VolcanoRuleMatchVisualizerListener visualizerListener =
+ * new VolcanoRuleMatchVisualizerListener(volcanoPlanner);
+ * volcanoPlanner.addListener(visualizerListener);
+ *
+ * volcanoPlanner.findBestExpr();
+ *
+ * // after the optimization, adds the final best plan
+ * visualizerListener.getVisualizer().addFinalPlan();
Review comment:
we could execute this step automatically when calling writeToFile(...)
##########
File path: core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRuleMatchVisualizer.java
##########
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.plan.volcano;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptCost;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.tools.visualizer.InputExcludedRelWriter;
+import org.apache.calcite.tools.visualizer.VisualizerNodeInfo;
+import org.apache.calcite.tools.visualizer.VisualizerRuleMatchInfo;
+
+import org.apache.commons.io.IOUtils;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Charsets;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+import static java.util.stream.Collectors.joining;
+
+/**
+ * This is tool to visualize the rule match process of the VolcanoPlanner.
+ *
+ *
+ * <p>To use the visualizer, add a listener before the VolcanoPlanner optimization phase.
+ * Then writes the output to a file after the optimization ends.
+ *
+ * <pre>
+ * // construct the visualizer and attach a listener to VolcanoPlanner
+ * VolcanoRuleMatchVisualizerListener visualizerListener =
+ * new VolcanoRuleMatchVisualizerListener(volcanoPlanner);
+ * volcanoPlanner.addListener(visualizerListener);
+ *
+ * volcanoPlanner.findBestExpr();
+ *
+ * // after the optimization, adds the final best plan
+ * visualizerListener.getVisualizer().addFinalPlan();
+ * // writes the output to files
+ * visualizerListener.getVisualizer().writeToFile(outputDirectory, "");
+ * </pre>
+ */
+public class VolcanoRuleMatchVisualizer {
+
+ VolcanoPlanner volcanoPlanner;
+
+ // a sequence of ruleMatch ID to represent the order of rule match
+ List<String> ruleMatchSequence = new ArrayList<>();
+ // map of ruleMatch ID and the info, including the state snapshot at the time of ruleMatch
+ Map<String, VisualizerRuleMatchInfo> ruleInfoMap = new HashMap<>();
+ // map of nodeID to the ruleID it's first added
+ Map<String, String> nodeAddedInRule = new HashMap<>();
+
+ // a map of relNode ID to the actual RelNode object
+ // contains all the relNodes appear during the optimization
+ // all RelNode are immutable in Calcite, therefore only new nodes will be added
+ Map<String, RelNode> allNodes = new HashMap<>();
+
+ public VolcanoRuleMatchVisualizer(VolcanoPlanner volcanoPlanner) {
+ this.volcanoPlanner = volcanoPlanner;
+ }
+
+ public void addRuleMatch(String ruleCallID, Collection<? extends RelNode> matchedRels) {
+
+ // store the current state snapshot
+ // nodes contained in the sets
+ // and inputs of relNodes (and relSubsets)
+ Map<String, String> setLabels = new HashMap<>();
+ Map<String, String> setOriginalRel = new HashMap<>();
+ Map<String, Set<String>> nodesInSet = new HashMap<>();
+ Map<String, Set<String>> nodeInputs = new HashMap<>();
+
+ // newNodes appeared after this ruleCall
+ Set<String> newNodes = new HashSet<>();
+
+ // populate current snapshot, and fill in the allNodes map
+ volcanoPlanner.allSets.forEach(set -> {
+ String setID = "set-" + set.id;
+ String setLabel = getSetLabel(set);
+ setLabels.put(setID, setLabel);
+ setOriginalRel.put(setID, set.rel == null ? "" : String.valueOf(set.rel.getId()));
+
+ nodesInSet.put(setID, nodesInSet.getOrDefault(setID, new HashSet<>()));
+
+ Consumer<RelNode> addNode = rel -> {
+ String nodeID = String.valueOf(rel.getId());
+ nodesInSet.get(setID).add(nodeID);
+
+ if (!allNodes.containsKey(nodeID)) {
+ newNodes.add(nodeID);
+ allNodes.put(nodeID, rel);
+ }
+ };
+
+ Consumer<RelNode> addLink = rel -> {
+ String nodeID = String.valueOf(rel.getId());
+ nodeInputs.put(nodeID, new HashSet<>());
+ if (rel instanceof RelSubset) {
+ RelSubset relSubset = (RelSubset) rel;
+ relSubset.getRelList().stream()
+ .filter(input -> input.getTraitSet().equals(relSubset.getTraitSet()))
+ .forEach(input -> nodeInputs.get(nodeID).add(String.valueOf(input.getId())));
Review comment:
What's the advantage over using `relSubset.getRels().forEach(input -> relInputs.add(String.valueOf(input.getId())));`?
##########
File path: core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRuleMatchVisualizer.java
##########
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.plan.volcano;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptCost;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.tools.visualizer.InputExcludedRelWriter;
+import org.apache.calcite.tools.visualizer.VisualizerNodeInfo;
+import org.apache.calcite.tools.visualizer.VisualizerRuleMatchInfo;
+
+import org.apache.commons.io.IOUtils;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Charsets;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+import static java.util.stream.Collectors.joining;
+
+/**
+ * This is tool to visualize the rule match process of the VolcanoPlanner.
+ *
+ *
+ * <p>To use the visualizer, add a listener before the VolcanoPlanner optimization phase.
+ * Then writes the output to a file after the optimization ends.
+ *
+ * <pre>
+ * // construct the visualizer and attach a listener to VolcanoPlanner
+ * VolcanoRuleMatchVisualizerListener visualizerListener =
+ * new VolcanoRuleMatchVisualizerListener(volcanoPlanner);
+ * volcanoPlanner.addListener(visualizerListener);
+ *
+ * volcanoPlanner.findBestExpr();
+ *
+ * // after the optimization, adds the final best plan
+ * visualizerListener.getVisualizer().addFinalPlan();
+ * // writes the output to files
+ * visualizerListener.getVisualizer().writeToFile(outputDirectory, "");
+ * </pre>
+ */
+public class VolcanoRuleMatchVisualizer {
+
+ VolcanoPlanner volcanoPlanner;
+
+ // a sequence of ruleMatch ID to represent the order of rule match
+ List<String> ruleMatchSequence = new ArrayList<>();
+ // map of ruleMatch ID and the info, including the state snapshot at the time of ruleMatch
+ Map<String, VisualizerRuleMatchInfo> ruleInfoMap = new HashMap<>();
+ // map of nodeID to the ruleID it's first added
+ Map<String, String> nodeAddedInRule = new HashMap<>();
+
+ // a map of relNode ID to the actual RelNode object
+ // contains all the relNodes appear during the optimization
+ // all RelNode are immutable in Calcite, therefore only new nodes will be added
+ Map<String, RelNode> allNodes = new HashMap<>();
+
+ public VolcanoRuleMatchVisualizer(VolcanoPlanner volcanoPlanner) {
+ this.volcanoPlanner = volcanoPlanner;
+ }
+
+ public void addRuleMatch(String ruleCallID, Collection<? extends RelNode> matchedRels) {
+
+ // store the current state snapshot
+ // nodes contained in the sets
+ // and inputs of relNodes (and relSubsets)
+ Map<String, String> setLabels = new HashMap<>();
+ Map<String, String> setOriginalRel = new HashMap<>();
+ Map<String, Set<String>> nodesInSet = new HashMap<>();
+ Map<String, Set<String>> nodeInputs = new HashMap<>();
+
+ // newNodes appeared after this ruleCall
+ Set<String> newNodes = new HashSet<>();
+
+ // populate current snapshot, and fill in the allNodes map
+ volcanoPlanner.allSets.forEach(set -> {
+ String setID = "set-" + set.id;
+ String setLabel = getSetLabel(set);
+ setLabels.put(setID, setLabel);
+ setOriginalRel.put(setID, set.rel == null ? "" : String.valueOf(set.rel.getId()));
+
+ nodesInSet.put(setID, nodesInSet.getOrDefault(setID, new HashSet<>()));
+
+ Consumer<RelNode> addNode = rel -> {
+ String nodeID = String.valueOf(rel.getId());
+ nodesInSet.get(setID).add(nodeID);
+
+ if (!allNodes.containsKey(nodeID)) {
+ newNodes.add(nodeID);
+ allNodes.put(nodeID, rel);
+ }
+ };
+
+ Consumer<RelNode> addLink = rel -> {
+ String nodeID = String.valueOf(rel.getId());
+ nodeInputs.put(nodeID, new HashSet<>());
+ if (rel instanceof RelSubset) {
+ RelSubset relSubset = (RelSubset) rel;
+ relSubset.getRelList().stream()
+ .filter(input -> input.getTraitSet().equals(relSubset.getTraitSet()))
+ .forEach(input -> nodeInputs.get(nodeID).add(String.valueOf(input.getId())));
+ relSubset.set.subsets.stream()
+ .filter(other -> !other.equals(relSubset))
+ .filter(other -> other.getTraitSet().satisfies(relSubset.getTraitSet()))
+ .forEach(other -> nodeInputs.get(nodeID).add(String.valueOf(other.getId())));
Review comment:
What's the advantage over using the following?
```
relSubset.getSubsetsSatisfyingThis()
.filter(other -> !other.equals(relSubset))
.forEach(input -> relInputs.add(String.valueOf(input.getId())));
```
##########
File path: core/src/main/resources/volcano-viz/viz-template.html
##########
@@ -0,0 +1,323 @@
+<!doctype html>
+<html lang="en">
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+<meta charset="utf-8">
+<title>Calcite Rule Match Visualization</title>
+
+<script src="https://d3js.org/d3.v5.min.js" charset="utf-8"></script>
+<script src="https://dagrejs.github.io/project/dagre-d3/latest/dagre-d3.min.js"></script>
+<script src="https://d3js.org/d3-zoom.v1.min.js"></script>
+<script src="https://unpkg.com/tippy.js@3/dist/tippy.all.min.js"></script>
+<script src="volcano-viz-data.js"></script>
+
+<style id="css">
+ body {
+ height: 100%;
+ margin: 0 0;
+ color: #333;
+ font-weight: 300;
+ font-family: "Helvetica Neue", Helvetica, Arial, sans-serf;
+ }
+
+ li a {
+ display: block;
+ /* and you can use padding for additional space if needs, as a clickable area / or other styling */
+ padding: 5px 20px;
+ }
+
+ section {
+ margin-bottom: 3em;
+ }
+
+ section p {
+ text-align: justify;
+ }
+
+ svg {
+ border: 1px solid #ccc;
+ overflow: hidden;
+ margin: 0 auto;
+ }
+
+ pre {
+ border: 1px solid #ccc;
+ }
+
+ .clusters rect {
+ fill: #FFFFE0;
+ stroke: #999;
+ stroke-width: 1.5px;
+ }
+
+ text {
+ font-weight: 300;
+ font-family: "Helvetica Neue", Helvetica, Arial, sans-serf;
+ font-size: 2em;
+ }
+
+ .node rect {
+ stroke: #999;
+ fill: #fff;
+ stroke-width: 1.5px;
+ }
+
+ .edgePath path {
+ stroke: #333;
+ stroke-width: 2px;
+ }
+
+ .container {
+ display: flex;
+ align-items: center;
+ }
+
+ .column1 {
+ flex: 0 0 300px;
+ }
+
+ .column2 {
+ flex: 0 0 1000px;
+ }
+
+ .tippy-content {
+ word-break: break-all;
+ word-wrap: break-word;
+ }
+</style>
+
+<div class="container">
+ <div class="column1">
+ <div style="width: 100%; text-align: center">
+ <div id="current-rule" style="display: block"></div>
+ <button id="prev-button" style="width: 80px; height: 40px; display:inline-block" disabled>prev </button>
+ <button id="next-button" style="width: 80px; height: 40px; display:inline-block" disabled> next</button>
+ </div>
+ <ul id="rule-match-list" style="width: 300px; height: 600px; overflow: auto">
+ </ul>
+ </div>
+ <div class="column2">
+ <svg id="svg-canvas" width="1200px" height="800px"></svg>
+ </div>
+</div>
+
+<script id="js">
+
+ var allNodes = data.allNodes;
+ var ruleMatchSequence = data.ruleMatchSequence;
+ var ruleMatchInfoMap = data.ruleMatchInfoMap;
+ var nodeAddedInRule = data.nodeAddedInRule;
+
+ /*
+ * Graph data and D3 JS render related variables
+ */
+
+ // Create the input graph
+ var g = new dagreD3.graphlib.Graph({
+ compound: true
+ })
+ .setGraph({
+ rankdir: 'LR'
+ })
+ .setDefaultEdgeLabel(function () {
+ return {};
+ });
+
+ // Create the renderer
+ var render = new dagreD3.render();
+
+ // Set up an SVG group so that we can translate the final graph.
+ var svg = d3.select("svg");
+ var svgGroup = svg.append("g");
+
+ // Set up zoom support
+ var svg = d3.select("svg")
+ .attr("width", "1200px")
+ .attr("height", "800px")
+ .call(d3.zoom().on("zoom", function () {
+ svgGroup.attr("transform", d3.event.transform)
+ }));
+
+ /*
+ * Global State
+ */
+
+ var currentRuleID = undefined;
+
+ /*
+ * Event Handler functions
+ */
+
+ var setCurrentRule = (ruleMatchID) => {
+ // un-highlight previous entry
+ var prevRuleID = currentRuleID;
+ if (prevRuleID !== undefined) {
+ var prevRuleElement = document.getElementById(prevRuleID);
+ prevRuleElement.style.backgroundColor = "#FFFFFF";
+ }
+
+ currentRuleID = ruleMatchID;
+ document.getElementById('current-rule').innerText = currentRuleID;
+
+ var currentRuleElement = document.getElementById(currentRuleID);
+ currentRuleElement.style.backgroundColor = "#D3D3D3";
+
+ var ruleIndex = ruleMatchSequence.indexOf(currentRuleID);
+
+ document.getElementById("prev-button").disabled = false;
+ document.getElementById("next-button").disabled = false;
Review comment:
We can set the correct value directly (without using an if).
##########
File path: core/src/main/resources/volcano-viz/viz-template.html
##########
@@ -0,0 +1,323 @@
+<!doctype html>
+<html lang="en">
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+<meta charset="utf-8">
+<title>Calcite Rule Match Visualization</title>
+
+<script src="https://d3js.org/d3.v5.min.js" charset="utf-8"></script>
Review comment:
Maybe we should switch to a more recent version. The latest version of d3 is 7.0.0. I tried v6 and v7 and this breaks only the panning and zooming. Probably not too difficult to fix.
##########
File path: core/src/main/resources/volcano-viz/volcano-viz-data.js
##########
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// sample generated visualizer data used by the visualizer JavaScript code
+var data = {
+
+ allNodes: {
+ "node1": {
+ label: "TableScan-1", // node label being displayed
+ isSubset: false, // if the node is a RelSubset
+ explanation: "table=t1", // additional explanation of properties
+ finalCost: "100 cpu, 100io" // final cost (in string) of the node at the end of optimization
+ },
+ "node2": {
+ label: "Filter-2",
+ isSubset: false,
+ explanation: "condition=c",
+ finalCost: "200cpu, 100io"
+ },
+ "node3": {
+ label: "TableSink-3",
+ isSubset: false,
+ explanation: "table=t2",
+ finalCost: "20cpu, 20io"
+ },
+ "node4": {
+ label: "IndexTableScan-4",
+ isSubset: false,
+ explanation: "table=t1, condition=c",
+ finalCost: "10cpu, 10io"
+ },
+ },
+
+ ruleMatchSequence: [
+ "INITIAL",
+ "IndexTableScanRule#1",
+ "FINAL"
+ ],
+
+ ruleMatchInfoMap: {
+ "INITIAL": {
+ setLabels: {
+ "set1": "set1",
+ "set2": "set2",
+ "set3": "set3",
+ },
+ setOriginalRel: {
+ "set1": "node1",
+ "set2": "node2",
+ "set3": "node3",
+ },
Review comment:
The setLabels and setOriginalRel could be moved to the root level (i.e., to line 47). This requires changing the js code, though.
##########
File path: core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRuleMatchVisualizer.java
##########
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.plan.volcano;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptCost;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.tools.visualizer.InputExcludedRelWriter;
+import org.apache.calcite.tools.visualizer.VisualizerNodeInfo;
+import org.apache.calcite.tools.visualizer.VisualizerRuleMatchInfo;
+
+import org.apache.commons.io.IOUtils;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Charsets;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+import static java.util.stream.Collectors.joining;
+
+/**
+ * This is tool to visualize the rule match process of the VolcanoPlanner.
+ *
+ *
+ * <p>To use the visualizer, add a listener before the VolcanoPlanner optimization phase.
+ * Then writes the output to a file after the optimization ends.
+ *
+ * <pre>
+ * // construct the visualizer and attach a listener to VolcanoPlanner
+ * VolcanoRuleMatchVisualizerListener visualizerListener =
+ * new VolcanoRuleMatchVisualizerListener(volcanoPlanner);
+ * volcanoPlanner.addListener(visualizerListener);
+ *
+ * volcanoPlanner.findBestExpr();
+ *
+ * // after the optimization, adds the final best plan
+ * visualizerListener.getVisualizer().addFinalPlan();
+ * // writes the output to files
+ * visualizerListener.getVisualizer().writeToFile(outputDirectory, "");
+ * </pre>
+ */
+public class VolcanoRuleMatchVisualizer {
+
+ VolcanoPlanner volcanoPlanner;
+
+ // a sequence of ruleMatch ID to represent the order of rule match
+ List<String> ruleMatchSequence = new ArrayList<>();
+ // map of ruleMatch ID and the info, including the state snapshot at the time of ruleMatch
+ Map<String, VisualizerRuleMatchInfo> ruleInfoMap = new HashMap<>();
+ // map of nodeID to the ruleID it's first added
+ Map<String, String> nodeAddedInRule = new HashMap<>();
+
+ // a map of relNode ID to the actual RelNode object
+ // contains all the relNodes appear during the optimization
+ // all RelNode are immutable in Calcite, therefore only new nodes will be added
+ Map<String, RelNode> allNodes = new HashMap<>();
+
+ public VolcanoRuleMatchVisualizer(VolcanoPlanner volcanoPlanner) {
+ this.volcanoPlanner = volcanoPlanner;
+ }
+
+ public void addRuleMatch(String ruleCallID, Collection<? extends RelNode> matchedRels) {
+
+ // store the current state snapshot
+ // nodes contained in the sets
+ // and inputs of relNodes (and relSubsets)
+ Map<String, String> setLabels = new HashMap<>();
+ Map<String, String> setOriginalRel = new HashMap<>();
+ Map<String, Set<String>> nodesInSet = new HashMap<>();
+ Map<String, Set<String>> nodeInputs = new HashMap<>();
Review comment:
Using TreeMaps would make the output more stable (i.e., no reordering when adding new nodes).
##########
File path: core/src/main/resources/volcano-viz/viz-template.html
##########
@@ -0,0 +1,323 @@
+<!doctype html>
+<html lang="en">
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+<meta charset="utf-8">
+<title>Calcite Rule Match Visualization</title>
+
+<script src="https://d3js.org/d3.v5.min.js" charset="utf-8"></script>
+<script src="https://dagrejs.github.io/project/dagre-d3/latest/dagre-d3.min.js"></script>
+<script src="https://d3js.org/d3-zoom.v1.min.js"></script>
+<script src="https://unpkg.com/tippy.js@3/dist/tippy.all.min.js"></script>
+<script src="volcano-viz-data.js"></script>
+
+<style id="css">
+ body {
+ height: 100%;
+ margin: 0 0;
+ color: #333;
+ font-weight: 300;
+ font-family: "Helvetica Neue", Helvetica, Arial, sans-serf;
+ }
+
+ li a {
+ display: block;
+ /* and you can use padding for additional space if needs, as a clickable area / or other styling */
+ padding: 5px 20px;
+ }
+
+ section {
+ margin-bottom: 3em;
+ }
+
+ section p {
+ text-align: justify;
+ }
+
+ svg {
+ border: 1px solid #ccc;
+ overflow: hidden;
+ margin: 0 auto;
+ }
+
+ pre {
+ border: 1px solid #ccc;
+ }
+
+ .clusters rect {
+ fill: #FFFFE0;
+ stroke: #999;
+ stroke-width: 1.5px;
+ }
+
+ text {
+ font-weight: 300;
+ font-family: "Helvetica Neue", Helvetica, Arial, sans-serf;
+ font-size: 2em;
+ }
+
+ .node rect {
+ stroke: #999;
+ fill: #fff;
+ stroke-width: 1.5px;
+ }
+
+ .edgePath path {
+ stroke: #333;
+ stroke-width: 2px;
+ }
+
+ .container {
+ display: flex;
+ align-items: center;
+ }
+
+ .column1 {
+ flex: 0 0 300px;
+ }
+
+ .column2 {
+ flex: 0 0 1000px;
+ }
+
+ .tippy-content {
+ word-break: break-all;
+ word-wrap: break-word;
+ }
+</style>
+
+<div class="container">
+ <div class="column1">
+ <div style="width: 100%; text-align: center">
+ <div id="current-rule" style="display: block"></div>
+ <button id="prev-button" style="width: 80px; height: 40px; display:inline-block" disabled>prev </button>
+ <button id="next-button" style="width: 80px; height: 40px; display:inline-block" disabled> next</button>
+ </div>
+ <ul id="rule-match-list" style="width: 300px; height: 600px; overflow: auto">
+ </ul>
+ </div>
+ <div class="column2">
+ <svg id="svg-canvas" width="1200px" height="800px"></svg>
+ </div>
+</div>
+
+<script id="js">
+
+ var allNodes = data.allNodes;
+ var ruleMatchSequence = data.ruleMatchSequence;
+ var ruleMatchInfoMap = data.ruleMatchInfoMap;
+ var nodeAddedInRule = data.nodeAddedInRule;
+
+ /*
+ * Graph data and D3 JS render related variables
+ */
+
+ // Create the input graph
+ var g = new dagreD3.graphlib.Graph({
+ compound: true
+ })
+ .setGraph({
+ rankdir: 'LR'
+ })
+ .setDefaultEdgeLabel(function () {
+ return {};
+ });
+
+ // Create the renderer
+ var render = new dagreD3.render();
+
+ // Set up an SVG group so that we can translate the final graph.
+ var svg = d3.select("svg");
+ var svgGroup = svg.append("g");
+
+ // Set up zoom support
+ var svg = d3.select("svg")
+ .attr("width", "1200px")
+ .attr("height", "800px")
+ .call(d3.zoom().on("zoom", function () {
+ svgGroup.attr("transform", d3.event.transform)
+ }));
+
+ /*
+ * Global State
+ */
+
+ var currentRuleID = undefined;
+
+ /*
+ * Event Handler functions
+ */
+
+ var setCurrentRule = (ruleMatchID) => {
+ // un-highlight previous entry
+ var prevRuleID = currentRuleID;
+ if (prevRuleID !== undefined) {
+ var prevRuleElement = document.getElementById(prevRuleID);
+ prevRuleElement.style.backgroundColor = "#FFFFFF";
+ }
+
+ currentRuleID = ruleMatchID;
+ document.getElementById('current-rule').innerText = currentRuleID;
+
+ var currentRuleElement = document.getElementById(currentRuleID);
+ currentRuleElement.style.backgroundColor = "#D3D3D3";
+
+ var ruleIndex = ruleMatchSequence.indexOf(currentRuleID);
+
+ document.getElementById("prev-button").disabled = false;
+ document.getElementById("next-button").disabled = false;
+
+ if (ruleIndex === 0) {
+ document.getElementById("prev-button").disabled = true;
+ }
+ if (ruleIndex === ruleMatchSequence.length - 1) {
+ document.getElementById("next-button").disabled = true;
+ }
+
+ createGraph(ruleMatchID);
+ }
+
+ var createGraph = (ruleMatchID) => {
+ var ruleMatchInfo = ruleMatchInfoMap[ruleMatchID]
+ console.log(ruleMatchInfo);
+
+ // remove previous rendered view and clear graph model
+ d3.select("svg g").selectAll("*").remove();
+ g.nodes().slice().forEach(nodeID => g.removeNode(nodeID));
+
+ // create nodes and sets
+ for (var setID in ruleMatchInfo.nodesInSet) {
+ // add set
+ var setLabel = ruleMatchInfo.setLabels[setID];
+ if (setLabel === null || setLabel === undefined) {
+ setLabel = setID;
+ }
+ g.setNode(setID, {
+ label: setLabel,
+ clusterLabelPos: 'top'
+ });
+ // add nodes and node-set parent relationship
+ var nodes = ruleMatchInfo.nodesInSet[setID];
+
+ nodes.forEach(nodeID => {
+ nodeInfo = allNodes[nodeID];
+ var nodeLabel;
+ if (ruleMatchID === "FINAL") {
+ nodeLabel = nodeInfo.label + "--" + nodeInfo.finalCost;
+ } else {
+ nodeLabel = nodeInfo.label;
+ }
+ var nodeStyle;
+ if (ruleMatchInfo.importanceZeroNodes
+ && ruleMatchInfo.importanceZeroNodes.includes(nodeID)) {
+ nodeStyle = "fill: #D3D3D3"
Review comment:
Using a class instead of explicit CSS attributes would allow to change the colors later. The same applies to the other styles set by the js code.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@calcite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [calcite] zuozhiw commented on a change in pull request #2486: [CALCITE-4737] Add Volcano Visualizer for Debugging
Posted by GitBox <gi...@apache.org>.
zuozhiw commented on a change in pull request #2486:
URL: https://github.com/apache/calcite/pull/2486#discussion_r689665526
##########
File path: core/src/main/resources/volcano-viz/viz-template.html
##########
@@ -0,0 +1,323 @@
+<!doctype html>
+<html lang="en">
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+<meta charset="utf-8">
+<title>Calcite Rule Match Visualization</title>
+
+<script src="https://d3js.org/d3.v5.min.js" charset="utf-8"></script>
+<script src="https://dagrejs.github.io/project/dagre-d3/latest/dagre-d3.min.js"></script>
+<script src="https://d3js.org/d3-zoom.v1.min.js"></script>
+<script src="https://unpkg.com/tippy.js@3/dist/tippy.all.min.js"></script>
+<script src="volcano-viz-data.js"></script>
+
+<style id="css">
+ body {
+ height: 100%;
+ margin: 0 0;
+ color: #333;
+ font-weight: 300;
+ font-family: "Helvetica Neue", Helvetica, Arial, sans-serf;
+ }
+
+ li a {
+ display: block;
+ /* and you can use padding for additional space if needs, as a clickable area / or other styling */
+ padding: 5px 20px;
+ }
+
+ section {
+ margin-bottom: 3em;
+ }
+
+ section p {
+ text-align: justify;
+ }
+
+ svg {
+ border: 1px solid #ccc;
+ overflow: hidden;
+ margin: 0 auto;
+ }
+
+ pre {
+ border: 1px solid #ccc;
+ }
+
+ .clusters rect {
+ fill: #FFFFE0;
+ stroke: #999;
+ stroke-width: 1.5px;
+ }
+
+ text {
+ font-weight: 300;
+ font-family: "Helvetica Neue", Helvetica, Arial, sans-serf;
+ font-size: 2em;
+ }
+
+ .node rect {
+ stroke: #999;
+ fill: #fff;
+ stroke-width: 1.5px;
+ }
+
+ .edgePath path {
+ stroke: #333;
+ stroke-width: 2px;
+ }
+
+ .container {
+ display: flex;
+ align-items: center;
+ }
+
+ .column1 {
+ flex: 0 0 300px;
+ }
+
+ .column2 {
+ flex: 0 0 1000px;
+ }
+
+ .tippy-content {
+ word-break: break-all;
+ word-wrap: break-word;
+ }
+</style>
+
+<div class="container">
+ <div class="column1">
+ <div style="width: 100%; text-align: center">
+ <div id="current-rule" style="display: block"></div>
+ <button id="prev-button" style="width: 80px; height: 40px; display:inline-block" disabled>prev </button>
+ <button id="next-button" style="width: 80px; height: 40px; display:inline-block" disabled> next</button>
+ </div>
+ <ul id="rule-match-list" style="width: 300px; height: 600px; overflow: auto">
+ </ul>
+ </div>
+ <div class="column2">
+ <svg id="svg-canvas" width="1200px" height="800px"></svg>
+ </div>
+</div>
+
+<script id="js">
+
+ var allNodes = data.allNodes;
+ var ruleMatchSequence = data.ruleMatchSequence;
+ var ruleMatchInfoMap = data.ruleMatchInfoMap;
+ var nodeAddedInRule = data.nodeAddedInRule;
+
+ /*
+ * Graph data and D3 JS render related variables
+ */
+
+ // Create the input graph
+ var g = new dagreD3.graphlib.Graph({
+ compound: true
+ })
+ .setGraph({
+ rankdir: 'LR'
+ })
+ .setDefaultEdgeLabel(function () {
+ return {};
+ });
+
+ // Create the renderer
+ var render = new dagreD3.render();
+
+ // Set up an SVG group so that we can translate the final graph.
+ var svg = d3.select("svg");
+ var svgGroup = svg.append("g");
+
+ // Set up zoom support
+ var svg = d3.select("svg")
+ .attr("width", "1200px")
+ .attr("height", "800px")
+ .call(d3.zoom().on("zoom", function () {
+ svgGroup.attr("transform", d3.event.transform)
+ }));
+
+ /*
+ * Global State
+ */
+
+ var currentRuleID = undefined;
+
+ /*
+ * Event Handler functions
+ */
+
+ var setCurrentRule = (ruleMatchID) => {
+ // un-highlight previous entry
+ var prevRuleID = currentRuleID;
+ if (prevRuleID !== undefined) {
+ var prevRuleElement = document.getElementById(prevRuleID);
+ prevRuleElement.style.backgroundColor = "#FFFFFF";
+ }
+
+ currentRuleID = ruleMatchID;
+ document.getElementById('current-rule').innerText = currentRuleID;
+
+ var currentRuleElement = document.getElementById(currentRuleID);
+ currentRuleElement.style.backgroundColor = "#D3D3D3";
+
+ var ruleIndex = ruleMatchSequence.indexOf(currentRuleID);
+
+ document.getElementById("prev-button").disabled = false;
+ document.getElementById("next-button").disabled = false;
+
+ if (ruleIndex === 0) {
+ document.getElementById("prev-button").disabled = true;
+ }
+ if (ruleIndex === ruleMatchSequence.length - 1) {
+ document.getElementById("next-button").disabled = true;
+ }
+
+ createGraph(ruleMatchID);
+ }
+
+ var createGraph = (ruleMatchID) => {
+ var ruleMatchInfo = ruleMatchInfoMap[ruleMatchID]
+ console.log(ruleMatchInfo);
+
+ // remove previous rendered view and clear graph model
+ d3.select("svg g").selectAll("*").remove();
+ g.nodes().slice().forEach(nodeID => g.removeNode(nodeID));
Review comment:
Agree, but we need to be careful and compare the old nodes with the new nodes. This could be tricky especially when set merge happens: one set is removed and relnodes from this set all go into another set. So re-drawing is eaiser.
One more thing on performance. This tool runs slow when the plan is ultra-large. I investigated and the major performance bottleneck is actually doing the **layout** instead of drawing the nodes. I can't recall the exact numbers but over 90% of the time is spent on the layout (`dagre`).
I did an extensive investigation on different layout libraries, but `dagre` remains the best-looking one, so I decided to keep it although it's slow. One sad thing is that none of these layout libraries can calculate layout incrementally. They all have to re-compute the whole thing every time.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@calcite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [calcite] zuozhiw commented on a change in pull request #2486: [CALCITE-4737] Add Volcano Visualizer for Debugging
Posted by GitBox <gi...@apache.org>.
zuozhiw commented on a change in pull request #2486:
URL: https://github.com/apache/calcite/pull/2486#discussion_r689656891
##########
File path: core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRuleMatchVisualizer.java
##########
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.plan.volcano;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptCost;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.tools.visualizer.InputExcludedRelWriter;
+import org.apache.calcite.tools.visualizer.VisualizerNodeInfo;
+import org.apache.calcite.tools.visualizer.VisualizerRuleMatchInfo;
+
+import org.apache.commons.io.IOUtils;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Charsets;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+import static java.util.stream.Collectors.joining;
+
+/**
+ * This is tool to visualize the rule match process of the VolcanoPlanner.
+ *
+ *
+ * <p>To use the visualizer, add a listener before the VolcanoPlanner optimization phase.
+ * Then writes the output to a file after the optimization ends.
+ *
+ * <pre>
+ * // construct the visualizer and attach a listener to VolcanoPlanner
+ * VolcanoRuleMatchVisualizerListener visualizerListener =
+ * new VolcanoRuleMatchVisualizerListener(volcanoPlanner);
+ * volcanoPlanner.addListener(visualizerListener);
+ *
+ * volcanoPlanner.findBestExpr();
+ *
+ * // after the optimization, adds the final best plan
+ * visualizerListener.getVisualizer().addFinalPlan();
+ * // writes the output to files
+ * visualizerListener.getVisualizer().writeToFile(outputDirectory, "");
+ * </pre>
+ */
+public class VolcanoRuleMatchVisualizer {
+
+ VolcanoPlanner volcanoPlanner;
+
+ // a sequence of ruleMatch ID to represent the order of rule match
+ List<String> ruleMatchSequence = new ArrayList<>();
+ // map of ruleMatch ID and the info, including the state snapshot at the time of ruleMatch
+ Map<String, VisualizerRuleMatchInfo> ruleInfoMap = new HashMap<>();
+ // map of nodeID to the ruleID it's first added
+ Map<String, String> nodeAddedInRule = new HashMap<>();
+
+ // a map of relNode ID to the actual RelNode object
+ // contains all the relNodes appear during the optimization
+ // all RelNode are immutable in Calcite, therefore only new nodes will be added
+ Map<String, RelNode> allNodes = new HashMap<>();
+
+ public VolcanoRuleMatchVisualizer(VolcanoPlanner volcanoPlanner) {
+ this.volcanoPlanner = volcanoPlanner;
+ }
+
+ public void addRuleMatch(String ruleCallID, Collection<? extends RelNode> matchedRels) {
+
+ // store the current state snapshot
+ // nodes contained in the sets
+ // and inputs of relNodes (and relSubsets)
+ Map<String, String> setLabels = new HashMap<>();
+ Map<String, String> setOriginalRel = new HashMap<>();
+ Map<String, Set<String>> nodesInSet = new HashMap<>();
+ Map<String, Set<String>> nodeInputs = new HashMap<>();
+
+ // newNodes appeared after this ruleCall
+ Set<String> newNodes = new HashSet<>();
+
+ // populate current snapshot, and fill in the allNodes map
+ volcanoPlanner.allSets.forEach(set -> {
+ String setID = "set-" + set.id;
+ String setLabel = getSetLabel(set);
+ setLabels.put(setID, setLabel);
+ setOriginalRel.put(setID, set.rel == null ? "" : String.valueOf(set.rel.getId()));
+
+ nodesInSet.put(setID, nodesInSet.getOrDefault(setID, new HashSet<>()));
+
+ Consumer<RelNode> addNode = rel -> {
+ String nodeID = String.valueOf(rel.getId());
+ nodesInSet.get(setID).add(nodeID);
+
+ if (!allNodes.containsKey(nodeID)) {
+ newNodes.add(nodeID);
+ allNodes.put(nodeID, rel);
+ }
+ };
+
+ Consumer<RelNode> addLink = rel -> {
+ String nodeID = String.valueOf(rel.getId());
+ nodeInputs.put(nodeID, new HashSet<>());
+ if (rel instanceof RelSubset) {
+ RelSubset relSubset = (RelSubset) rel;
+ relSubset.getRelList().stream()
+ .filter(input -> input.getTraitSet().equals(relSubset.getTraitSet()))
+ .forEach(input -> nodeInputs.get(nodeID).add(String.valueOf(input.getId())));
Review comment:
This was a design tradeoff to make the links in a set less messy. In the environment I was working on, a set contains multiple subsets with related traits. So a relNode belongs to many subsets at the same time.
For example, suppose there are 2 subsets `S1`, `S2` (`S2` satisfies `S1`) and 2 relnodes `R1`, `R2` (with the same trait as `S2`).
Instead of creating 4 links for relNodes: `R1 -> S1`, `R1 -> S2`, `R2 -> S1`, `R2 -> S2`,
I decided to create 2 links for relNodes: `R1 -> S2` and `R2 -> S2`, and 1 indirect link for the subsets `S2 -> S1`. So the subset satisfaction relationship is indirectly shown.
This way in a set with many subsets/relnodes, the links are much less messy.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@calcite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [calcite] zuozhiw commented on pull request #2486: [CALCITE-4737] Add Volcano Visualizer for Debugging
Posted by GitBox <gi...@apache.org>.
zuozhiw commented on pull request #2486:
URL: https://github.com/apache/calcite/pull/2486#issuecomment-951588529
closed in favor of #2552
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@calcite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [calcite] zuozhiw commented on a change in pull request #2486: [CALCITE-4737] Add Volcano Visualizer for Debugging
Posted by GitBox <gi...@apache.org>.
zuozhiw commented on a change in pull request #2486:
URL: https://github.com/apache/calcite/pull/2486#discussion_r689665526
##########
File path: core/src/main/resources/volcano-viz/viz-template.html
##########
@@ -0,0 +1,323 @@
+<!doctype html>
+<html lang="en">
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+<meta charset="utf-8">
+<title>Calcite Rule Match Visualization</title>
+
+<script src="https://d3js.org/d3.v5.min.js" charset="utf-8"></script>
+<script src="https://dagrejs.github.io/project/dagre-d3/latest/dagre-d3.min.js"></script>
+<script src="https://d3js.org/d3-zoom.v1.min.js"></script>
+<script src="https://unpkg.com/tippy.js@3/dist/tippy.all.min.js"></script>
+<script src="volcano-viz-data.js"></script>
+
+<style id="css">
+ body {
+ height: 100%;
+ margin: 0 0;
+ color: #333;
+ font-weight: 300;
+ font-family: "Helvetica Neue", Helvetica, Arial, sans-serf;
+ }
+
+ li a {
+ display: block;
+ /* and you can use padding for additional space if needs, as a clickable area / or other styling */
+ padding: 5px 20px;
+ }
+
+ section {
+ margin-bottom: 3em;
+ }
+
+ section p {
+ text-align: justify;
+ }
+
+ svg {
+ border: 1px solid #ccc;
+ overflow: hidden;
+ margin: 0 auto;
+ }
+
+ pre {
+ border: 1px solid #ccc;
+ }
+
+ .clusters rect {
+ fill: #FFFFE0;
+ stroke: #999;
+ stroke-width: 1.5px;
+ }
+
+ text {
+ font-weight: 300;
+ font-family: "Helvetica Neue", Helvetica, Arial, sans-serf;
+ font-size: 2em;
+ }
+
+ .node rect {
+ stroke: #999;
+ fill: #fff;
+ stroke-width: 1.5px;
+ }
+
+ .edgePath path {
+ stroke: #333;
+ stroke-width: 2px;
+ }
+
+ .container {
+ display: flex;
+ align-items: center;
+ }
+
+ .column1 {
+ flex: 0 0 300px;
+ }
+
+ .column2 {
+ flex: 0 0 1000px;
+ }
+
+ .tippy-content {
+ word-break: break-all;
+ word-wrap: break-word;
+ }
+</style>
+
+<div class="container">
+ <div class="column1">
+ <div style="width: 100%; text-align: center">
+ <div id="current-rule" style="display: block"></div>
+ <button id="prev-button" style="width: 80px; height: 40px; display:inline-block" disabled>prev </button>
+ <button id="next-button" style="width: 80px; height: 40px; display:inline-block" disabled> next</button>
+ </div>
+ <ul id="rule-match-list" style="width: 300px; height: 600px; overflow: auto">
+ </ul>
+ </div>
+ <div class="column2">
+ <svg id="svg-canvas" width="1200px" height="800px"></svg>
+ </div>
+</div>
+
+<script id="js">
+
+ var allNodes = data.allNodes;
+ var ruleMatchSequence = data.ruleMatchSequence;
+ var ruleMatchInfoMap = data.ruleMatchInfoMap;
+ var nodeAddedInRule = data.nodeAddedInRule;
+
+ /*
+ * Graph data and D3 JS render related variables
+ */
+
+ // Create the input graph
+ var g = new dagreD3.graphlib.Graph({
+ compound: true
+ })
+ .setGraph({
+ rankdir: 'LR'
+ })
+ .setDefaultEdgeLabel(function () {
+ return {};
+ });
+
+ // Create the renderer
+ var render = new dagreD3.render();
+
+ // Set up an SVG group so that we can translate the final graph.
+ var svg = d3.select("svg");
+ var svgGroup = svg.append("g");
+
+ // Set up zoom support
+ var svg = d3.select("svg")
+ .attr("width", "1200px")
+ .attr("height", "800px")
+ .call(d3.zoom().on("zoom", function () {
+ svgGroup.attr("transform", d3.event.transform)
+ }));
+
+ /*
+ * Global State
+ */
+
+ var currentRuleID = undefined;
+
+ /*
+ * Event Handler functions
+ */
+
+ var setCurrentRule = (ruleMatchID) => {
+ // un-highlight previous entry
+ var prevRuleID = currentRuleID;
+ if (prevRuleID !== undefined) {
+ var prevRuleElement = document.getElementById(prevRuleID);
+ prevRuleElement.style.backgroundColor = "#FFFFFF";
+ }
+
+ currentRuleID = ruleMatchID;
+ document.getElementById('current-rule').innerText = currentRuleID;
+
+ var currentRuleElement = document.getElementById(currentRuleID);
+ currentRuleElement.style.backgroundColor = "#D3D3D3";
+
+ var ruleIndex = ruleMatchSequence.indexOf(currentRuleID);
+
+ document.getElementById("prev-button").disabled = false;
+ document.getElementById("next-button").disabled = false;
+
+ if (ruleIndex === 0) {
+ document.getElementById("prev-button").disabled = true;
+ }
+ if (ruleIndex === ruleMatchSequence.length - 1) {
+ document.getElementById("next-button").disabled = true;
+ }
+
+ createGraph(ruleMatchID);
+ }
+
+ var createGraph = (ruleMatchID) => {
+ var ruleMatchInfo = ruleMatchInfoMap[ruleMatchID]
+ console.log(ruleMatchInfo);
+
+ // remove previous rendered view and clear graph model
+ d3.select("svg g").selectAll("*").remove();
+ g.nodes().slice().forEach(nodeID => g.removeNode(nodeID));
Review comment:
Agree, but we need to be careful and compare the old nodes with the new nodes. This could be tricky especially when set merge happens: one set is removed and relnodes from this set all go into another set. So re-drawing is eaiser.
One more thing on performance. This tool runs slow when the plan is ultra-large. I investigated and the major performance bottleneck is actually doing the **layout** instead of drawing the nodes. I can't recall the numbers but over 90% of the time is spent on the layout (`dagre`).
I did an extensive investigation on different layout libraries, but `dagre` remains the best-looking one, so I decided to keep it although it's slow. One sad thing is that none of these layout libraries can do it incrementally.
##########
File path: core/src/main/resources/volcano-viz/viz-template.html
##########
@@ -0,0 +1,323 @@
+<!doctype html>
+<html lang="en">
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+<meta charset="utf-8">
+<title>Calcite Rule Match Visualization</title>
+
+<script src="https://d3js.org/d3.v5.min.js" charset="utf-8"></script>
+<script src="https://dagrejs.github.io/project/dagre-d3/latest/dagre-d3.min.js"></script>
+<script src="https://d3js.org/d3-zoom.v1.min.js"></script>
+<script src="https://unpkg.com/tippy.js@3/dist/tippy.all.min.js"></script>
+<script src="volcano-viz-data.js"></script>
+
+<style id="css">
+ body {
+ height: 100%;
+ margin: 0 0;
+ color: #333;
+ font-weight: 300;
+ font-family: "Helvetica Neue", Helvetica, Arial, sans-serf;
+ }
+
+ li a {
+ display: block;
+ /* and you can use padding for additional space if needs, as a clickable area / or other styling */
+ padding: 5px 20px;
+ }
+
+ section {
+ margin-bottom: 3em;
+ }
+
+ section p {
+ text-align: justify;
+ }
+
+ svg {
+ border: 1px solid #ccc;
+ overflow: hidden;
+ margin: 0 auto;
+ }
+
+ pre {
+ border: 1px solid #ccc;
+ }
+
+ .clusters rect {
+ fill: #FFFFE0;
+ stroke: #999;
+ stroke-width: 1.5px;
+ }
+
+ text {
+ font-weight: 300;
+ font-family: "Helvetica Neue", Helvetica, Arial, sans-serf;
+ font-size: 2em;
+ }
+
+ .node rect {
+ stroke: #999;
+ fill: #fff;
+ stroke-width: 1.5px;
+ }
+
+ .edgePath path {
+ stroke: #333;
+ stroke-width: 2px;
+ }
+
+ .container {
+ display: flex;
+ align-items: center;
+ }
+
+ .column1 {
+ flex: 0 0 300px;
+ }
+
+ .column2 {
+ flex: 0 0 1000px;
+ }
+
+ .tippy-content {
+ word-break: break-all;
+ word-wrap: break-word;
+ }
+</style>
+
+<div class="container">
+ <div class="column1">
+ <div style="width: 100%; text-align: center">
+ <div id="current-rule" style="display: block"></div>
+ <button id="prev-button" style="width: 80px; height: 40px; display:inline-block" disabled>prev </button>
+ <button id="next-button" style="width: 80px; height: 40px; display:inline-block" disabled> next</button>
+ </div>
+ <ul id="rule-match-list" style="width: 300px; height: 600px; overflow: auto">
+ </ul>
+ </div>
+ <div class="column2">
+ <svg id="svg-canvas" width="1200px" height="800px"></svg>
+ </div>
+</div>
+
+<script id="js">
+
+ var allNodes = data.allNodes;
+ var ruleMatchSequence = data.ruleMatchSequence;
+ var ruleMatchInfoMap = data.ruleMatchInfoMap;
+ var nodeAddedInRule = data.nodeAddedInRule;
+
+ /*
+ * Graph data and D3 JS render related variables
+ */
+
+ // Create the input graph
+ var g = new dagreD3.graphlib.Graph({
+ compound: true
+ })
+ .setGraph({
+ rankdir: 'LR'
+ })
+ .setDefaultEdgeLabel(function () {
+ return {};
+ });
+
+ // Create the renderer
+ var render = new dagreD3.render();
+
+ // Set up an SVG group so that we can translate the final graph.
+ var svg = d3.select("svg");
+ var svgGroup = svg.append("g");
+
+ // Set up zoom support
+ var svg = d3.select("svg")
+ .attr("width", "1200px")
+ .attr("height", "800px")
+ .call(d3.zoom().on("zoom", function () {
+ svgGroup.attr("transform", d3.event.transform)
+ }));
+
+ /*
+ * Global State
+ */
+
+ var currentRuleID = undefined;
+
+ /*
+ * Event Handler functions
+ */
+
+ var setCurrentRule = (ruleMatchID) => {
+ // un-highlight previous entry
+ var prevRuleID = currentRuleID;
+ if (prevRuleID !== undefined) {
+ var prevRuleElement = document.getElementById(prevRuleID);
+ prevRuleElement.style.backgroundColor = "#FFFFFF";
+ }
+
+ currentRuleID = ruleMatchID;
+ document.getElementById('current-rule').innerText = currentRuleID;
+
+ var currentRuleElement = document.getElementById(currentRuleID);
+ currentRuleElement.style.backgroundColor = "#D3D3D3";
+
+ var ruleIndex = ruleMatchSequence.indexOf(currentRuleID);
+
+ document.getElementById("prev-button").disabled = false;
+ document.getElementById("next-button").disabled = false;
+
+ if (ruleIndex === 0) {
+ document.getElementById("prev-button").disabled = true;
+ }
+ if (ruleIndex === ruleMatchSequence.length - 1) {
+ document.getElementById("next-button").disabled = true;
+ }
+
+ createGraph(ruleMatchID);
+ }
+
+ var createGraph = (ruleMatchID) => {
+ var ruleMatchInfo = ruleMatchInfoMap[ruleMatchID]
+ console.log(ruleMatchInfo);
+
+ // remove previous rendered view and clear graph model
+ d3.select("svg g").selectAll("*").remove();
+ g.nodes().slice().forEach(nodeID => g.removeNode(nodeID));
Review comment:
Agree, but we need to be careful and compare the old nodes with the new nodes. This could be tricky especially when set merge happens: one set is removed and relnodes from this set all go into another set. So re-drawing is eaiser.
One more thing on performance. This tool runs slow when the plan is ultra-large. I investigated and the major performance bottleneck is actually doing the **layout** instead of drawing the nodes. I can't recall the exact numbers but over 90% of the time is spent on the layout (`dagre`).
I did an extensive investigation on different layout libraries, but `dagre` remains the best-looking one, so I decided to keep it although it's slow. One sad thing is that none of these layout libraries can do it incrementally.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@calcite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [calcite] zuozhiw commented on a change in pull request #2486: [CALCITE-4737] Add Volcano Visualizer for Debugging
Posted by GitBox <gi...@apache.org>.
zuozhiw commented on a change in pull request #2486:
URL: https://github.com/apache/calcite/pull/2486#discussion_r689660349
##########
File path: core/src/main/resources/volcano-viz/viz-template.html
##########
@@ -0,0 +1,323 @@
+<!doctype html>
+<html lang="en">
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+<meta charset="utf-8">
+<title>Calcite Rule Match Visualization</title>
+
+<script src="https://d3js.org/d3.v5.min.js" charset="utf-8"></script>
Review comment:
agree, it's been some time since I implemented this code, so the library might be out of date, but one thing to be careful is the compatibility with dagre-d3, which seems to be not actively maintained anymore https://github.com/dagrejs/dagre-d3
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@calcite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [calcite] zuozhiw commented on a change in pull request #2486: [CALCITE-4737] Add Volcano Visualizer for Debugging
Posted by GitBox <gi...@apache.org>.
zuozhiw commented on a change in pull request #2486:
URL: https://github.com/apache/calcite/pull/2486#discussion_r689666693
##########
File path: core/src/main/resources/volcano-viz/viz-template.html
##########
@@ -0,0 +1,323 @@
+<!doctype html>
+<html lang="en">
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+<meta charset="utf-8">
+<title>Calcite Rule Match Visualization</title>
+
+<script src="https://d3js.org/d3.v5.min.js" charset="utf-8"></script>
+<script src="https://dagrejs.github.io/project/dagre-d3/latest/dagre-d3.min.js"></script>
+<script src="https://d3js.org/d3-zoom.v1.min.js"></script>
+<script src="https://unpkg.com/tippy.js@3/dist/tippy.all.min.js"></script>
+<script src="volcano-viz-data.js"></script>
+
+<style id="css">
+ body {
+ height: 100%;
+ margin: 0 0;
+ color: #333;
+ font-weight: 300;
+ font-family: "Helvetica Neue", Helvetica, Arial, sans-serf;
+ }
+
+ li a {
+ display: block;
+ /* and you can use padding for additional space if needs, as a clickable area / or other styling */
+ padding: 5px 20px;
+ }
+
+ section {
+ margin-bottom: 3em;
+ }
+
+ section p {
+ text-align: justify;
+ }
+
+ svg {
+ border: 1px solid #ccc;
+ overflow: hidden;
+ margin: 0 auto;
+ }
+
+ pre {
+ border: 1px solid #ccc;
+ }
+
+ .clusters rect {
+ fill: #FFFFE0;
+ stroke: #999;
+ stroke-width: 1.5px;
+ }
+
+ text {
+ font-weight: 300;
+ font-family: "Helvetica Neue", Helvetica, Arial, sans-serf;
+ font-size: 2em;
+ }
+
+ .node rect {
+ stroke: #999;
+ fill: #fff;
+ stroke-width: 1.5px;
+ }
+
+ .edgePath path {
+ stroke: #333;
+ stroke-width: 2px;
+ }
+
+ .container {
+ display: flex;
+ align-items: center;
+ }
+
+ .column1 {
+ flex: 0 0 300px;
+ }
+
+ .column2 {
+ flex: 0 0 1000px;
+ }
+
+ .tippy-content {
+ word-break: break-all;
+ word-wrap: break-word;
+ }
+</style>
+
+<div class="container">
+ <div class="column1">
+ <div style="width: 100%; text-align: center">
+ <div id="current-rule" style="display: block"></div>
+ <button id="prev-button" style="width: 80px; height: 40px; display:inline-block" disabled>prev </button>
+ <button id="next-button" style="width: 80px; height: 40px; display:inline-block" disabled> next</button>
+ </div>
+ <ul id="rule-match-list" style="width: 300px; height: 600px; overflow: auto">
+ </ul>
+ </div>
+ <div class="column2">
+ <svg id="svg-canvas" width="1200px" height="800px"></svg>
+ </div>
+</div>
+
+<script id="js">
+
+ var allNodes = data.allNodes;
+ var ruleMatchSequence = data.ruleMatchSequence;
+ var ruleMatchInfoMap = data.ruleMatchInfoMap;
+ var nodeAddedInRule = data.nodeAddedInRule;
+
+ /*
+ * Graph data and D3 JS render related variables
+ */
+
+ // Create the input graph
+ var g = new dagreD3.graphlib.Graph({
+ compound: true
+ })
+ .setGraph({
+ rankdir: 'LR'
+ })
+ .setDefaultEdgeLabel(function () {
+ return {};
+ });
+
+ // Create the renderer
+ var render = new dagreD3.render();
+
+ // Set up an SVG group so that we can translate the final graph.
+ var svg = d3.select("svg");
+ var svgGroup = svg.append("g");
+
+ // Set up zoom support
+ var svg = d3.select("svg")
+ .attr("width", "1200px")
+ .attr("height", "800px")
+ .call(d3.zoom().on("zoom", function () {
+ svgGroup.attr("transform", d3.event.transform)
+ }));
+
+ /*
+ * Global State
+ */
+
+ var currentRuleID = undefined;
+
+ /*
+ * Event Handler functions
+ */
+
+ var setCurrentRule = (ruleMatchID) => {
+ // un-highlight previous entry
+ var prevRuleID = currentRuleID;
+ if (prevRuleID !== undefined) {
+ var prevRuleElement = document.getElementById(prevRuleID);
+ prevRuleElement.style.backgroundColor = "#FFFFFF";
+ }
+
+ currentRuleID = ruleMatchID;
+ document.getElementById('current-rule').innerText = currentRuleID;
+
+ var currentRuleElement = document.getElementById(currentRuleID);
+ currentRuleElement.style.backgroundColor = "#D3D3D3";
+
+ var ruleIndex = ruleMatchSequence.indexOf(currentRuleID);
+
+ document.getElementById("prev-button").disabled = false;
+ document.getElementById("next-button").disabled = false;
+
+ if (ruleIndex === 0) {
+ document.getElementById("prev-button").disabled = true;
+ }
+ if (ruleIndex === ruleMatchSequence.length - 1) {
+ document.getElementById("next-button").disabled = true;
+ }
+
+ createGraph(ruleMatchID);
+ }
+
+ var createGraph = (ruleMatchID) => {
+ var ruleMatchInfo = ruleMatchInfoMap[ruleMatchID]
+ console.log(ruleMatchInfo);
+
+ // remove previous rendered view and clear graph model
+ d3.select("svg g").selectAll("*").remove();
+ g.nodes().slice().forEach(nodeID => g.removeNode(nodeID));
+
+ // create nodes and sets
+ for (var setID in ruleMatchInfo.nodesInSet) {
+ // add set
+ var setLabel = ruleMatchInfo.setLabels[setID];
+ if (setLabel === null || setLabel === undefined) {
+ setLabel = setID;
+ }
+ g.setNode(setID, {
+ label: setLabel,
+ clusterLabelPos: 'top'
+ });
+ // add nodes and node-set parent relationship
+ var nodes = ruleMatchInfo.nodesInSet[setID];
+
+ nodes.forEach(nodeID => {
+ nodeInfo = allNodes[nodeID];
+ var nodeLabel;
+ if (ruleMatchID === "FINAL") {
+ nodeLabel = nodeInfo.label + "--" + nodeInfo.finalCost;
+ } else {
+ nodeLabel = nodeInfo.label;
+ }
+ var nodeStyle;
+ if (ruleMatchInfo.importanceZeroNodes
+ && ruleMatchInfo.importanceZeroNodes.includes(nodeID)) {
+ nodeStyle = "fill: #D3D3D3"
Review comment:
agree
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@calcite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [calcite] zuozhiw commented on a change in pull request #2486: [CALCITE-4737] Add Volcano Visualizer for Debugging
Posted by GitBox <gi...@apache.org>.
zuozhiw commented on a change in pull request #2486:
URL: https://github.com/apache/calcite/pull/2486#discussion_r689656891
##########
File path: core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRuleMatchVisualizer.java
##########
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.plan.volcano;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptCost;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.tools.visualizer.InputExcludedRelWriter;
+import org.apache.calcite.tools.visualizer.VisualizerNodeInfo;
+import org.apache.calcite.tools.visualizer.VisualizerRuleMatchInfo;
+
+import org.apache.commons.io.IOUtils;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Charsets;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+import static java.util.stream.Collectors.joining;
+
+/**
+ * This is tool to visualize the rule match process of the VolcanoPlanner.
+ *
+ *
+ * <p>To use the visualizer, add a listener before the VolcanoPlanner optimization phase.
+ * Then writes the output to a file after the optimization ends.
+ *
+ * <pre>
+ * // construct the visualizer and attach a listener to VolcanoPlanner
+ * VolcanoRuleMatchVisualizerListener visualizerListener =
+ * new VolcanoRuleMatchVisualizerListener(volcanoPlanner);
+ * volcanoPlanner.addListener(visualizerListener);
+ *
+ * volcanoPlanner.findBestExpr();
+ *
+ * // after the optimization, adds the final best plan
+ * visualizerListener.getVisualizer().addFinalPlan();
+ * // writes the output to files
+ * visualizerListener.getVisualizer().writeToFile(outputDirectory, "");
+ * </pre>
+ */
+public class VolcanoRuleMatchVisualizer {
+
+ VolcanoPlanner volcanoPlanner;
+
+ // a sequence of ruleMatch ID to represent the order of rule match
+ List<String> ruleMatchSequence = new ArrayList<>();
+ // map of ruleMatch ID and the info, including the state snapshot at the time of ruleMatch
+ Map<String, VisualizerRuleMatchInfo> ruleInfoMap = new HashMap<>();
+ // map of nodeID to the ruleID it's first added
+ Map<String, String> nodeAddedInRule = new HashMap<>();
+
+ // a map of relNode ID to the actual RelNode object
+ // contains all the relNodes appear during the optimization
+ // all RelNode are immutable in Calcite, therefore only new nodes will be added
+ Map<String, RelNode> allNodes = new HashMap<>();
+
+ public VolcanoRuleMatchVisualizer(VolcanoPlanner volcanoPlanner) {
+ this.volcanoPlanner = volcanoPlanner;
+ }
+
+ public void addRuleMatch(String ruleCallID, Collection<? extends RelNode> matchedRels) {
+
+ // store the current state snapshot
+ // nodes contained in the sets
+ // and inputs of relNodes (and relSubsets)
+ Map<String, String> setLabels = new HashMap<>();
+ Map<String, String> setOriginalRel = new HashMap<>();
+ Map<String, Set<String>> nodesInSet = new HashMap<>();
+ Map<String, Set<String>> nodeInputs = new HashMap<>();
+
+ // newNodes appeared after this ruleCall
+ Set<String> newNodes = new HashSet<>();
+
+ // populate current snapshot, and fill in the allNodes map
+ volcanoPlanner.allSets.forEach(set -> {
+ String setID = "set-" + set.id;
+ String setLabel = getSetLabel(set);
+ setLabels.put(setID, setLabel);
+ setOriginalRel.put(setID, set.rel == null ? "" : String.valueOf(set.rel.getId()));
+
+ nodesInSet.put(setID, nodesInSet.getOrDefault(setID, new HashSet<>()));
+
+ Consumer<RelNode> addNode = rel -> {
+ String nodeID = String.valueOf(rel.getId());
+ nodesInSet.get(setID).add(nodeID);
+
+ if (!allNodes.containsKey(nodeID)) {
+ newNodes.add(nodeID);
+ allNodes.put(nodeID, rel);
+ }
+ };
+
+ Consumer<RelNode> addLink = rel -> {
+ String nodeID = String.valueOf(rel.getId());
+ nodeInputs.put(nodeID, new HashSet<>());
+ if (rel instanceof RelSubset) {
+ RelSubset relSubset = (RelSubset) rel;
+ relSubset.getRelList().stream()
+ .filter(input -> input.getTraitSet().equals(relSubset.getTraitSet()))
+ .forEach(input -> nodeInputs.get(nodeID).add(String.valueOf(input.getId())));
Review comment:
This was a design tradeoff to make the links in a set less messy. In the environment I was working on, a set contains multiple subsets with related traits (mainly different distributions and collations). So a relNode belongs to many subsets at the same time.
For example, suppose there are 2 subsets `S1`, `S2` (`S2` satisfies `S1`) and 2 relnodes `R1`, `R2` (with the same trait as `S2`).
Instead of creating 4 links for relNodes: `R1 -> S1`, `R1 -> S2`, `R2 -> S1`, `R2 -> S2`,
I decided to create 2 links for relNodes: `R1 -> S2` and `R2 -> S2`, and 1 link for the subsets `S2 -> S1`. So the subset satisfaction relationship is indirectly shown.
This way in a set with many subsets/relnodes, the links are much less messy.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@calcite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [calcite] thomasrebele commented on pull request #2486: [CALCITE-4737] Add Volcano Visualizer for Debugging
Posted by GitBox <gi...@apache.org>.
thomasrebele commented on pull request #2486:
URL: https://github.com/apache/calcite/pull/2486#issuecomment-899409149
Thank you for the pull request. I fixed the errors reported by CheckerFramework and ErrorProne in my fork. See thomasrebele/calcite/tree/CALCITE-4737.
Generally it looks good. The attributes setLabels and setOriginalRel are the same for all steps. In the json file I would move them to the root level (after allNodes). I'll also add some minor review comments.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@calcite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [calcite] zuozhiw commented on a change in pull request #2486: [CALCITE-4737] Add Volcano Visualizer for Debugging
Posted by GitBox <gi...@apache.org>.
zuozhiw commented on a change in pull request #2486:
URL: https://github.com/apache/calcite/pull/2486#discussion_r689658773
##########
File path: core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRuleMatchVisualizer.java
##########
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.plan.volcano;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptCost;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.tools.visualizer.InputExcludedRelWriter;
+import org.apache.calcite.tools.visualizer.VisualizerNodeInfo;
+import org.apache.calcite.tools.visualizer.VisualizerRuleMatchInfo;
+
+import org.apache.commons.io.IOUtils;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Charsets;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+import static java.util.stream.Collectors.joining;
+
+/**
+ * This is tool to visualize the rule match process of the VolcanoPlanner.
+ *
+ *
+ * <p>To use the visualizer, add a listener before the VolcanoPlanner optimization phase.
+ * Then writes the output to a file after the optimization ends.
+ *
+ * <pre>
+ * // construct the visualizer and attach a listener to VolcanoPlanner
+ * VolcanoRuleMatchVisualizerListener visualizerListener =
+ * new VolcanoRuleMatchVisualizerListener(volcanoPlanner);
+ * volcanoPlanner.addListener(visualizerListener);
+ *
+ * volcanoPlanner.findBestExpr();
+ *
+ * // after the optimization, adds the final best plan
+ * visualizerListener.getVisualizer().addFinalPlan();
+ * // writes the output to files
+ * visualizerListener.getVisualizer().writeToFile(outputDirectory, "");
+ * </pre>
+ */
+public class VolcanoRuleMatchVisualizer {
+
+ VolcanoPlanner volcanoPlanner;
+
+ // a sequence of ruleMatch ID to represent the order of rule match
+ List<String> ruleMatchSequence = new ArrayList<>();
+ // map of ruleMatch ID and the info, including the state snapshot at the time of ruleMatch
+ Map<String, VisualizerRuleMatchInfo> ruleInfoMap = new HashMap<>();
+ // map of nodeID to the ruleID it's first added
+ Map<String, String> nodeAddedInRule = new HashMap<>();
+
+ // a map of relNode ID to the actual RelNode object
+ // contains all the relNodes appear during the optimization
+ // all RelNode are immutable in Calcite, therefore only new nodes will be added
+ Map<String, RelNode> allNodes = new HashMap<>();
+
+ public VolcanoRuleMatchVisualizer(VolcanoPlanner volcanoPlanner) {
+ this.volcanoPlanner = volcanoPlanner;
+ }
+
+ public void addRuleMatch(String ruleCallID, Collection<? extends RelNode> matchedRels) {
+
+ // store the current state snapshot
+ // nodes contained in the sets
+ // and inputs of relNodes (and relSubsets)
+ Map<String, String> setLabels = new HashMap<>();
+ Map<String, String> setOriginalRel = new HashMap<>();
+ Map<String, Set<String>> nodesInSet = new HashMap<>();
+ Map<String, Set<String>> nodeInputs = new HashMap<>();
+
+ // newNodes appeared after this ruleCall
+ Set<String> newNodes = new HashSet<>();
+
+ // populate current snapshot, and fill in the allNodes map
+ volcanoPlanner.allSets.forEach(set -> {
+ String setID = "set-" + set.id;
+ String setLabel = getSetLabel(set);
+ setLabels.put(setID, setLabel);
+ setOriginalRel.put(setID, set.rel == null ? "" : String.valueOf(set.rel.getId()));
+
+ nodesInSet.put(setID, nodesInSet.getOrDefault(setID, new HashSet<>()));
+
+ Consumer<RelNode> addNode = rel -> {
+ String nodeID = String.valueOf(rel.getId());
+ nodesInSet.get(setID).add(nodeID);
+
+ if (!allNodes.containsKey(nodeID)) {
+ newNodes.add(nodeID);
+ allNodes.put(nodeID, rel);
+ }
+ };
+
+ Consumer<RelNode> addLink = rel -> {
+ String nodeID = String.valueOf(rel.getId());
+ nodeInputs.put(nodeID, new HashSet<>());
+ if (rel instanceof RelSubset) {
+ RelSubset relSubset = (RelSubset) rel;
+ relSubset.getRelList().stream()
+ .filter(input -> input.getTraitSet().equals(relSubset.getTraitSet()))
+ .forEach(input -> nodeInputs.get(nodeID).add(String.valueOf(input.getId())));
+ relSubset.set.subsets.stream()
+ .filter(other -> !other.equals(relSubset))
+ .filter(other -> other.getTraitSet().satisfies(relSubset.getTraitSet()))
+ .forEach(other -> nodeInputs.get(nodeID).add(String.valueOf(other.getId())));
Review comment:
`getSubsetsSatisfyingThis` seems to be a new api that wasn't there when I wrote this code, I agree we should use it
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@calcite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org