You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@calcite.apache.org by jh...@apache.org on 2021/03/28 04:13:16 UTC
[calcite] branch master updated (4bc9166 -> f4a5512)
This is an automated email from the ASF dual-hosted git repository.
jhyde pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/calcite.git.
from 4bc9166 Add Matcher#matches to ForbiddenApis to avoid its accidental use
new 1721825 [CALCITE-4552] Interpreter does not close resources held by its Nodes on close
new f4a5512 [CALCITE-4522] CPU cost of Sort should be lower if sort keys are empty (huangqixiang)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../adapter/enumerable/EnumerableLimitSort.java | 32 -----
.../calcite/interpreter/AbstractSingleNode.java | 4 +
.../apache/calcite/interpreter/Interpreter.java | 9 ++
.../org/apache/calcite/interpreter/JoinNode.java | 6 +-
.../java/org/apache/calcite/interpreter/Node.java | 5 +-
.../org/apache/calcite/interpreter/SetOpNode.java | 5 +
.../java/org/apache/calcite/rel/core/Sort.java | 71 +++++++++-
.../calcite/rel/metadata/BuiltInMetadata.java | 1 +
.../main/java/org/apache/calcite/util/Util.java | 9 ++
.../org/apache/calcite/test/InterpreterTest.java | 34 ++++-
.../java/org/apache/calcite/test/JdbcTest.java | 8 +-
.../org/apache/calcite/test/RelMetadataTest.java | 151 +++++++++++++++++----
.../apache/calcite/test/ScannableTableTest.java | 49 ++++---
.../apache/calcite/adapter/tpcds/TpcdsTest.java | 6 +-
14 files changed, 296 insertions(+), 94 deletions(-)
[calcite] 01/02: [CALCITE-4552] Interpreter does not close
resources held by its Nodes on close
Posted by jh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jhyde pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/calcite.git
commit 1721825eabfc4a0f82b33711b159359ee3c4341a
Author: Julian Hyde <jh...@apache.org>
AuthorDate: Wed Mar 24 18:11:56 2021 -0700
[CALCITE-4552] Interpreter does not close resources held by its Nodes on close
---
.../calcite/interpreter/AbstractSingleNode.java | 4 ++
.../apache/calcite/interpreter/Interpreter.java | 9 ++++
.../org/apache/calcite/interpreter/JoinNode.java | 6 ++-
.../java/org/apache/calcite/interpreter/Node.java | 5 ++-
.../org/apache/calcite/interpreter/SetOpNode.java | 5 +++
.../org/apache/calcite/test/InterpreterTest.java | 34 +++++++++++----
.../apache/calcite/test/ScannableTableTest.java | 49 +++++++++++++++-------
7 files changed, 87 insertions(+), 25 deletions(-)
diff --git a/core/src/main/java/org/apache/calcite/interpreter/AbstractSingleNode.java b/core/src/main/java/org/apache/calcite/interpreter/AbstractSingleNode.java
index 007ee92..9d328bb 100644
--- a/core/src/main/java/org/apache/calcite/interpreter/AbstractSingleNode.java
+++ b/core/src/main/java/org/apache/calcite/interpreter/AbstractSingleNode.java
@@ -33,4 +33,8 @@ abstract class AbstractSingleNode<T extends SingleRel> implements Node {
this.source = compiler.source(rel, 0);
this.sink = compiler.sink(rel);
}
+
+ @Override public void close() {
+ source.close();
+ }
}
diff --git a/core/src/main/java/org/apache/calcite/interpreter/Interpreter.java b/core/src/main/java/org/apache/calcite/interpreter/Interpreter.java
index fe1f4ab..58b0427 100644
--- a/core/src/main/java/org/apache/calcite/interpreter/Interpreter.java
+++ b/core/src/main/java/org/apache/calcite/interpreter/Interpreter.java
@@ -146,6 +146,7 @@ public class Interpreter extends AbstractEnumerable<@Nullable Object[]>
}
@Override public void close() {
+ nodes.values().forEach(NodeInfo::close);
}
/** Not used. */
@@ -269,6 +270,14 @@ public class Interpreter extends AbstractEnumerable<@Nullable Object[]>
this.rel = rel;
this.rowEnumerable = rowEnumerable;
}
+
+ void close() {
+ if (node != null) {
+ final Node n = node;
+ node = null;
+ n.close();
+ }
+ }
}
/**
diff --git a/core/src/main/java/org/apache/calcite/interpreter/JoinNode.java b/core/src/main/java/org/apache/calcite/interpreter/JoinNode.java
index 6d88213..9fc63f7 100644
--- a/core/src/main/java/org/apache/calcite/interpreter/JoinNode.java
+++ b/core/src/main/java/org/apache/calcite/interpreter/JoinNode.java
@@ -53,8 +53,12 @@ public class JoinNode implements Node {
}
- @Override public void run() throws InterruptedException {
+ @Override public void close() {
+ leftSource.close();
+ rightSource.close();
+ }
+ @Override public void run() throws InterruptedException {
final int fieldCount = rel.getLeft().getRowType().getFieldCount()
+ rel.getRight().getRowType().getFieldCount();
context.values = new Object[fieldCount];
diff --git a/core/src/main/java/org/apache/calcite/interpreter/Node.java b/core/src/main/java/org/apache/calcite/interpreter/Node.java
index 330f043..94bcf08 100644
--- a/core/src/main/java/org/apache/calcite/interpreter/Node.java
+++ b/core/src/main/java/org/apache/calcite/interpreter/Node.java
@@ -19,6 +19,9 @@ package org.apache.calcite.interpreter;
/**
* Relational expression that can be executed using an interpreter.
*/
-public interface Node {
+public interface Node extends AutoCloseable {
void run() throws InterruptedException;
+
+ @Override default void close() {
+ }
}
diff --git a/core/src/main/java/org/apache/calcite/interpreter/SetOpNode.java b/core/src/main/java/org/apache/calcite/interpreter/SetOpNode.java
index 1775b34..2d5750e 100644
--- a/core/src/main/java/org/apache/calcite/interpreter/SetOpNode.java
+++ b/core/src/main/java/org/apache/calcite/interpreter/SetOpNode.java
@@ -43,6 +43,11 @@ public class SetOpNode implements Node {
this.setOp = setOp;
}
+ @Override public void close() {
+ leftSource.close();
+ rightSource.close();
+ }
+
@Override public void run() throws InterruptedException {
final Collection<Row> leftRows;
final Collection<Row> rightRows;
diff --git a/core/src/test/java/org/apache/calcite/test/InterpreterTest.java b/core/src/test/java/org/apache/calcite/test/InterpreterTest.java
index 1b95cc4..4c3ade9 100644
--- a/core/src/test/java/org/apache/calcite/test/InterpreterTest.java
+++ b/core/src/test/java/org/apache/calcite/test/InterpreterTest.java
@@ -54,10 +54,12 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Function;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.Is.is;
/**
* Unit tests for {@link org.apache.calcite.interpreter.Interpreter}.
@@ -201,13 +203,14 @@ class InterpreterTest {
private static void assertInterpret(RelNode rel, DataContext dataContext,
boolean unordered, String... rows) {
- final Interpreter interpreter = new Interpreter(dataContext, rel);
- final List<RelDataType> fieldTypes =
- Util.transform(rel.getRowType().getFieldList(),
- RelDataTypeField::getType);
- assertRows(interpreter,
- EnumUtils.toExternal(fieldTypes, DateTimeUtils.DEFAULT_ZONE), unordered,
- rows);
+ try (Interpreter interpreter = new Interpreter(dataContext, rel)) {
+ final List<RelDataType> fieldTypes =
+ Util.transform(rel.getRowType().getFieldList(),
+ RelDataTypeField::getType);
+ final Function<@Nullable Object[], List<@Nullable Object>> converter =
+ EnumUtils.toExternal(fieldTypes, DateTimeUtils.DEFAULT_ZONE);
+ assertRows(interpreter, converter, unordered, rows);
+ }
}
private static void assertRows(Interpreter interpreter,
@@ -242,6 +245,23 @@ class InterpreterTest {
.returnsRows("[4, John]", "[4, Paul]", "[5, Ringo]", "[6, George]");
}
+ /** Tests executing a plan on a
+ * {@link org.apache.calcite.schema.ScannableTable} using an interpreter. */
+ @Test void testInterpretScannableTable2() {
+ final AtomicInteger scanCount = new AtomicInteger();
+ final AtomicInteger enumerateCount = new AtomicInteger();
+ final AtomicInteger closeCount = new AtomicInteger();
+ rootSchema.add("counting",
+ ScannableTableTest.countingTable(scanCount, enumerateCount,
+ closeCount));
+ sql("select * from \"counting\" order by \"i\"")
+ .returnsRows("[0]", "[10]", "[20]", "[30]");
+ assertThat(scanCount.get(), is(1));
+ assertThat(enumerateCount.get(), is(1));
+ assertThat("close is called twice: on last fetch, and interpreter close",
+ closeCount.get(), is(2));
+ }
+
@Test void testAggregateCount() {
rootSchema.add("beatles", new ScannableTableTest.BeatlesTable());
sql("select count(*) from \"beatles\"")
diff --git a/core/src/test/java/org/apache/calcite/test/ScannableTableTest.java b/core/src/test/java/org/apache/calcite/test/ScannableTableTest.java
index 6294931..df19ddb 100644
--- a/core/src/test/java/org/apache/calcite/test/ScannableTableTest.java
+++ b/core/src/test/java/org/apache/calcite/test/ScannableTableTest.java
@@ -19,6 +19,7 @@ package org.apache.calcite.test;
import org.apache.calcite.DataContext;
import org.apache.calcite.jdbc.CalciteConnection;
import org.apache.calcite.linq4j.AbstractEnumerable;
+import org.apache.calcite.linq4j.DelegatingEnumerator;
import org.apache.calcite.linq4j.Enumerable;
import org.apache.calcite.linq4j.Enumerator;
import org.apache.calcite.rel.type.RelDataType;
@@ -45,6 +46,7 @@ import org.apache.calcite.util.Pair;
import com.google.common.collect.ImmutableMap;
import org.checkerframework.checker.nullness.qual.Nullable;
+import org.jetbrains.annotations.NotNull;
import org.junit.jupiter.api.Test;
import java.math.BigDecimal;
@@ -407,26 +409,12 @@ public class ScannableTableTest {
final AtomicInteger scanCount = new AtomicInteger();
final AtomicInteger enumerateCount = new AtomicInteger();
+ final AtomicInteger closeCount = new AtomicInteger();
final Schema schema =
new AbstractSchema() {
@Override protected Map<String, Table> getTableMap() {
return ImmutableMap.of("TENS",
- new SimpleTable() {
- private Enumerable<Object[]> superScan(DataContext root) {
- return super.scan(root);
- }
-
- @Override public Enumerable<@Nullable Object[]>
- scan(final DataContext root) {
- scanCount.incrementAndGet();
- return new AbstractEnumerable<Object[]>() {
- public Enumerator<Object[]> enumerator() {
- enumerateCount.incrementAndGet();
- return superScan(root).enumerator();
- }
- };
- }
- });
+ countingTable(scanCount, enumerateCount, closeCount));
}
};
calciteConnection.getRootSchema().add("TEST", schema);
@@ -626,6 +614,35 @@ public class ScannableTableTest {
};
}
+ /** Returns a table that counts the number of calls to
+ * {@link ScannableTable#scan}, {@link Enumerable#enumerator()},
+ * and {@link Enumerator#close()}. */
+ static SimpleTable countingTable(AtomicInteger scanCount,
+ AtomicInteger enumerateCount, AtomicInteger closeCount) {
+ return new SimpleTable() {
+ private Enumerable<Object[]> superScan(DataContext root) {
+ return super.scan(root);
+ }
+
+ @Override public Enumerable<@Nullable Object[]> scan(DataContext root) {
+ scanCount.incrementAndGet();
+ return new AbstractEnumerable<Object[]>() {
+ @NotNull @Override public Enumerator<Object[]> enumerator() {
+ enumerateCount.incrementAndGet();
+ final Enumerator<Object[]> enumerator =
+ superScan(root).enumerator();
+ return new DelegatingEnumerator<Object[]>(enumerator) {
+ @Override public void close() {
+ closeCount.incrementAndGet();
+ super.close();
+ }
+ };
+ }
+ };
+ }
+ };
+ }
+
private static final Object[][] BEATLES = {
{4, "John", 1940},
{4, "Paul", 1942},
[calcite] 02/02: [CALCITE-4522] CPU cost of Sort should be lower if
sort keys are empty (huangqixiang)
Posted by jh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jhyde pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/calcite.git
commit f4a5512caeaf087147fec82b7d98ea231b134036
Author: huangqixiang <hu...@kuaishou.mail>
AuthorDate: Wed Mar 10 21:08:25 2021 +0800
[CALCITE-4522] CPU cost of Sort should be lower if sort keys are empty (huangqixiang)
The model is as follows:
* If fetch is zero, CPU cost is zero; otherwise,
* if the sort keys are empty, we don't need to sort, only step over
the rows, and therefore the CPU cost is
min(fetch + offset, inputRowCount) * bytesPerRow; otherwise
* we need to read and sort inputRowCount rows, with at most
min(fetch + offset, inputRowCount) of them in the sort data
structure at a time, giving a CPU cost of
inputRowCount * log(min(fetch + offset, inputRowCount)) * bytesPerRow.
The cost model factors in row width via bytesPerRow, because
sorts need to move rows around, not just compare them; by making the cost
higher if rows are wider, we discourage pushing a Project through a Sort.
We assume that each field is 4 bytes, and we add 3 'virtual fields' to
represent the per-row overhead. Thus a 1-field row is (3 + 1) * 4 = 16
bytes; a 5-field row is (3 + 5) * 4 = 32 bytes.
The cost model does not consider a 5-field sort to be more expensive
than, say, a 2-field sort, because both sorts will compare just one field
most of the time.
Close apache/calcite#2363
---
.../adapter/enumerable/EnumerableLimitSort.java | 32 -----
.../java/org/apache/calcite/rel/core/Sort.java | 71 +++++++++-
.../calcite/rel/metadata/BuiltInMetadata.java | 1 +
.../main/java/org/apache/calcite/util/Util.java | 9 ++
.../java/org/apache/calcite/test/JdbcTest.java | 8 +-
.../org/apache/calcite/test/RelMetadataTest.java | 151 +++++++++++++++++----
.../apache/calcite/adapter/tpcds/TpcdsTest.java | 6 +-
7 files changed, 209 insertions(+), 69 deletions(-)
diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSort.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSort.java
index 28f02a4..0ae9723 100644
--- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSort.java
+++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSort.java
@@ -20,15 +20,10 @@ import org.apache.calcite.linq4j.tree.BlockBuilder;
import org.apache.calcite.linq4j.tree.Expression;
import org.apache.calcite.linq4j.tree.Expressions;
import org.apache.calcite.plan.RelOptCluster;
-import org.apache.calcite.plan.RelOptCost;
-import org.apache.calcite.plan.RelOptPlanner;
import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.rel.RelCollation;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Sort;
-import org.apache.calcite.rel.metadata.RelMetadataQuery;
-import org.apache.calcite.rex.RexDynamicParam;
-import org.apache.calcite.rex.RexLiteral;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.util.BuiltInMethod;
import org.apache.calcite.util.Pair;
@@ -128,31 +123,4 @@ public class EnumerableLimitSort extends Sort implements EnumerableRel {
)));
return implementor.result(physType, builder.toBlock());
}
-
- @Override public @Nullable RelOptCost computeSelfCost(RelOptPlanner planner,
- RelMetadataQuery mq) {
- final double rowCount = mq.getRowCount(this.input).doubleValue();
- double toSort = getValue(this.fetch, rowCount);
- if (this.offset != null) {
- toSort += getValue(this.offset, rowCount);
- }
- // we need to sort at most rowCount rows
- toSort = Math.min(rowCount, toSort);
-
- // we need to process rowCount rows, and for every row
- // we search the key in a TreeMap with at most toSort entries
- final double lookup = Math.max(1., Math.log(toSort));
- final double bytesPerRow = this.getRowType().getFieldCount() * 4.;
- final double cpu = (rowCount * lookup) * bytesPerRow;
-
- RelOptCost cost = planner.getCostFactory().makeCost(rowCount, cpu, 0);
- return cost;
- }
-
- private static double getValue(@Nullable RexNode r, double defaultValue) {
- if (r == null || r instanceof RexDynamicParam) {
- return defaultValue;
- }
- return RexLiteral.intValue(r);
- }
}
diff --git a/core/src/main/java/org/apache/calcite/rel/core/Sort.java b/core/src/main/java/org/apache/calcite/rel/core/Sort.java
index 0ce2089..1faad39 100644
--- a/core/src/main/java/org/apache/calcite/rel/core/Sort.java
+++ b/core/src/main/java/org/apache/calcite/rel/core/Sort.java
@@ -29,6 +29,7 @@ import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.RelWriter;
import org.apache.calcite.rel.SingleRel;
import org.apache.calcite.rel.metadata.RelMetadataQuery;
+import org.apache.calcite.rex.RexLiteral;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexShuttle;
import org.apache.calcite.util.Util;
@@ -122,14 +123,65 @@ public abstract class Sort extends SingleRel {
public abstract Sort copy(RelTraitSet traitSet, RelNode newInput,
RelCollation newCollation, @Nullable RexNode offset, @Nullable RexNode fetch);
+ /** {@inheritDoc}
+ *
+ * <p>The CPU cost of a Sort has three main cases:
+ *
+ * <ul>
+ * <li>If {@code fetch} is zero, CPU cost is zero; otherwise,
+ *
+ * <li>if the sort keys are empty, we don't need to sort, only step over
+ * the rows, and therefore the CPU cost is
+ * {@code min(fetch + offset, inputRowCount) * bytesPerRow}; otherwise
+ *
+ * <li>we need to read and sort {@code inputRowCount} rows, with at most
+ * {@code min(fetch + offset, inputRowCount)} of them in the sort data
+ * structure at a time, giving a CPU cost of {@code inputRowCount *
+ * log(min(fetch + offset, inputRowCount)) * bytesPerRow}.
+ * </ul>
+ *
+ * <p>The cost model factors in row width via {@code bytesPerRow}, because
+ * sorts need to move rows around, not just compare them; by making the cost
+ * higher if rows are wider, we discourage pushing a Project through a Sort.
+ * We assume that each field is 4 bytes, and we add 3 'virtual fields' to
+ * represent the per-row overhead. Thus a 1-field row is (3 + 1) * 4 = 16
+ * bytes; a 5-field row is (3 + 5) * 4 = 32 bytes.
+ *
+ * <p>The cost model does not consider a 5-field sort to be more expensive
+ * than, say, a 2-field sort, because both sorts will compare just one field
+ * most of the time. */
@Override public @Nullable RelOptCost computeSelfCost(RelOptPlanner planner,
RelMetadataQuery mq) {
- // Higher cost if rows are wider discourages pushing a project through a
- // sort.
- final double rowCount = mq.getRowCount(this);
- final double bytesPerRow = getRowType().getFieldCount() * 4;
- final double cpu = Util.nLogN(rowCount) * bytesPerRow;
- return planner.getCostFactory().makeCost(rowCount, cpu, 0);
+ final double offsetValue = Util.first(doubleValue(offset), 0d);
+ assert offsetValue >= 0 : "offset should not be negative:" + offsetValue;
+
+ final double inCount = mq.getRowCount(input);
+ @Nullable Double fetchValue = doubleValue(fetch);
+ final double readCount;
+ if (fetchValue == null) {
+ readCount = inCount;
+ } else if (fetchValue <= 0) {
+ // Case 1. Read zero rows from input, therefore CPU cost is zero.
+ return planner.getCostFactory().makeCost(inCount, 0, 0);
+ } else {
+ readCount = Math.min(inCount, offsetValue + fetchValue);
+ }
+
+ final double bytesPerRow = (3 + getRowType().getFieldCount()) * 4;
+
+ final double cpu;
+ if (collation.getFieldCollations().isEmpty()) {
+ // Case 2. If sort keys are empty, CPU cost is cheaper because we are just
+ // stepping over the first "readCount" rows, rather than sorting all
+ // "inCount" them. (Presumably we are applying FETCH and/or OFFSET,
+ // otherwise this Sort is a no-op.)
+ cpu = readCount * bytesPerRow;
+ } else {
+ // Case 3. Read and sort all "inCount" rows, keeping "readCount" in the
+ // sort data structure at a time.
+ cpu = Util.nLogM(inCount, readCount) * bytesPerRow;
+ }
+ return planner.getCostFactory().makeCost(readCount, cpu, 0);
}
@Override public RelNode accept(RexShuttle shuttle) {
@@ -193,4 +245,11 @@ public abstract class Sort extends SingleRel {
pw.itemIf("fetch", fetch, fetch != null);
return pw;
}
+
+ /** Returns the double value of a node if it is a literal, otherwise null. */
+ private static @Nullable Double doubleValue(@Nullable RexNode r) {
+ return r instanceof RexLiteral
+ ? ((RexLiteral) r).getValueAs(Double.class)
+ : null;
+ }
}
diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/BuiltInMetadata.java b/core/src/main/java/org/apache/calcite/rel/metadata/BuiltInMetadata.java
index 92e242a..614cc15 100644
--- a/core/src/main/java/org/apache/calcite/rel/metadata/BuiltInMetadata.java
+++ b/core/src/main/java/org/apache/calcite/rel/metadata/BuiltInMetadata.java
@@ -17,6 +17,7 @@
package org.apache.calcite.rel.metadata;
import org.apache.calcite.plan.RelOptCost;
+import org.apache.calcite.plan.RelOptPlanner;
import org.apache.calcite.plan.RelOptPredicateList;
import org.apache.calcite.plan.volcano.VolcanoPlanner;
import org.apache.calcite.rel.RelCollation;
diff --git a/core/src/main/java/org/apache/calcite/util/Util.java b/core/src/main/java/org/apache/calcite/util/Util.java
index 15b2fd4..5f77a4d 100644
--- a/core/src/main/java/org/apache/calcite/util/Util.java
+++ b/core/src/main/java/org/apache/calcite/util/Util.java
@@ -370,6 +370,15 @@ public class Util {
}
/**
+ * Computes <code>nlog(m)</code> using the natural logarithm (or <code>
+ * n</code> if <code>m < {@link Math#E}</code>, so the result is never
+ * negative.
+ */
+ public static double nLogM(double n, double m) {
+ return (m < Math.E) ? n : (n * Math.log(m));
+ }
+
+ /**
* Prints an object using reflection. We can handle <code>null</code>;
* arrays of objects and primitive values; for regular objects, we print all
* public fields.
diff --git a/core/src/test/java/org/apache/calcite/test/JdbcTest.java b/core/src/test/java/org/apache/calcite/test/JdbcTest.java
index 45b0ab0..96b2829 100644
--- a/core/src/test/java/org/apache/calcite/test/JdbcTest.java
+++ b/core/src/test/java/org/apache/calcite/test/JdbcTest.java
@@ -1162,10 +1162,10 @@ public class JdbcTest {
+ "and p.\"brand_name\" = 'Washington'")
.explainMatches("including all attributes ",
CalciteAssert.checkMaskedResultContains(""
- + "EnumerableMergeJoin(condition=[=($0, $38)], joinType=[inner]): rowcount = 7.050660528307499E8, cumulative cost = {7.656040129282498E8 rows, 5.0023949296644424E10 cpu, 0.0 io}\n"
- + " EnumerableSort(sort0=[$0], dir0=[ASC]): rowcount = 2.0087351932499997E7, cumulative cost = {4.044858016499999E7 rows, 5.0023896255644424E10 cpu, 0.0 io}\n"
- + " EnumerableMergeJoin(condition=[=($2, $8)], joinType=[inner]): rowcount = 2.0087351932499997E7, cumulative cost = {2.0361228232499994E7 rows, 3.232400376004586E7 cpu, 0.0 io}\n"
- + " EnumerableSort(sort0=[$2], dir0=[ASC]): rowcount = 86837.0, cumulative cost = {173674.0 rows, 3.168658076004586E7 cpu, 0.0 io}\n"
+ + "EnumerableMergeJoin(condition=[=($0, $38)], joinType=[inner]): rowcount = 7.050660528307499E8, cumulative cost = {7.656040129282498E8 rows, 5.408916992330521E10 cpu, 0.0 io}\n"
+ + " EnumerableSort(sort0=[$0], dir0=[ASC]): rowcount = 2.0087351932499997E7, cumulative cost = {4.044858016499999E7 rows, 5.408911688230521E10 cpu, 0.0 io}\n"
+ + " EnumerableMergeJoin(condition=[=($2, $8)], joinType=[inner]): rowcount = 2.0087351932499997E7, cumulative cost = {2.0361228232499994E7 rows, 4.4173907295063056E7 cpu, 0.0 io}\n"
+ + " EnumerableSort(sort0=[$2], dir0=[ASC]): rowcount = 86837.0, cumulative cost = {173674.0 rows, 4.3536484295063056E7 cpu, 0.0 io}\n"
+ " EnumerableTableScan(table=[[foodmart2, sales_fact_1997]]): rowcount = 86837.0, cumulative cost = {86837.0 rows, 86838.0 cpu, 0.0 io}\n"
+ " EnumerableCalc(expr#0..28=[{inputs}], expr#29=['San Francisco':VARCHAR(30)], expr#30=[=($t9, $t29)], proj#0..28=[{exprs}], $condition=[$t30]): rowcount = 1542.1499999999999, cumulative cost = {11823.15 rows, 637423.0 cpu, 0.0 io}\n"
+ " EnumerableTableScan(table=[[foodmart2, customer]]): rowcount = 10281.0, cumulative cost = {10281.0 rows, 10282.0 cpu, 0.0 io}\n"
diff --git a/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java b/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java
index 00edfcf..7d84dbb 100644
--- a/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java
+++ b/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java
@@ -20,6 +20,7 @@ import org.apache.calcite.adapter.enumerable.EnumerableMergeJoin;
import org.apache.calcite.config.CalciteSystemProperty;
import org.apache.calcite.linq4j.tree.Types;
import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptCost;
import org.apache.calcite.plan.RelOptPlanner;
import org.apache.calcite.plan.RelOptPredicateList;
import org.apache.calcite.plan.RelOptTable;
@@ -28,6 +29,7 @@ import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.plan.hep.HepPlanner;
import org.apache.calcite.plan.hep.HepProgram;
import org.apache.calcite.plan.hep.HepProgramBuilder;
+import org.apache.calcite.plan.volcano.VolcanoPlanner;
import org.apache.calcite.rel.RelCollation;
import org.apache.calcite.rel.RelCollationTraitDef;
import org.apache.calcite.rel.RelCollations;
@@ -91,6 +93,7 @@ import org.apache.calcite.rex.RexProgram;
import org.apache.calcite.rex.RexTableInputRef;
import org.apache.calcite.rex.RexTableInputRef.RelTableRef;
import org.apache.calcite.runtime.SqlFunctions;
+import org.apache.calcite.sql.SqlExplainLevel;
import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.SqlOperator;
import org.apache.calcite.sql.SqlSpecialOperator;
@@ -107,6 +110,7 @@ import org.apache.calcite.util.BuiltInMethod;
import org.apache.calcite.util.Holder;
import org.apache.calcite.util.ImmutableBitSet;
import org.apache.calcite.util.ImmutableIntList;
+import org.apache.calcite.util.Util;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
@@ -192,6 +196,11 @@ public class RelMetadataTest extends SqlToRelTestBase {
//~ Methods ----------------------------------------------------------------
+ /** Creates a tester. */
+ Sql sql(String sql) {
+ return new Sql(tester, sql);
+ }
+
// ----------------------------------------------------------------------
// Tests for getPercentageOriginalRows
// ----------------------------------------------------------------------
@@ -532,6 +541,33 @@ public class RelMetadataTest extends SqlToRelTestBase {
false);
}
+ /** Test case for
+ * <a href="https://issues.apache.org/jira/browse/CALCITE-4192">[CALCITE-4192]
+ * RelMdColumnOrigins get the wrong index of group by columns after RelNode
+ * was optimized by AggregateProjectMergeRule rule</a>. */
+ @Test void testColumnOriginAfterAggProjectMergeRule() {
+ final String sql = "select count(ename), SAL from emp group by SAL";
+ final RelNode rel = tester.convertSqlToRel(sql).rel;
+ final HepProgramBuilder programBuilder = HepProgram.builder();
+ programBuilder.addRuleInstance(CoreRules.AGGREGATE_PROJECT_MERGE);
+ final HepPlanner planner = new HepPlanner(programBuilder.build());
+ planner.setRoot(rel);
+ final RelNode optimizedRel = planner.findBestExp();
+
+ Set<RelColumnOrigin> origins = RelMetadataQuery.instance()
+ .getColumnOrigins(optimizedRel, 1);
+ assertThat(origins.size(), equalTo(1));
+
+ RelColumnOrigin columnOrigin = origins.iterator().next();
+ assertThat(columnOrigin.getOriginColumnOrdinal(), equalTo(5));
+ assertThat(columnOrigin.getOriginTable().getRowType().getFieldNames().get(5),
+ equalTo("SAL"));
+ }
+
+ // ----------------------------------------------------------------------
+ // Tests for getRowCount, getMinRowCount, getMaxRowCount
+ // ----------------------------------------------------------------------
+
private void checkRowCount(String sql, double expected, double expectedMin,
double expectedMax) {
RelNode rel = convertSql(sql);
@@ -644,7 +680,6 @@ public class RelMetadataTest extends SqlToRelTestBase {
0D, 0D); // 0 * 4
}
-
@Test void testRowCountJoinEmptyEmpty() {
final String sql = "select * from (select * from emp limit 0) as emp\n"
+ "inner join (select * from dept limit 0) as dept\n"
@@ -808,6 +843,69 @@ public class RelMetadataTest extends SqlToRelTestBase {
checkRowCount(sql, 1D, 1D, 1D);
}
+ // ----------------------------------------------------------------------
+ // Tests for computeSelfCost.cpu
+ // ----------------------------------------------------------------------
+
+ @Test void testSortCpuCostOffsetLimit() {
+ final String sql = "select ename, deptno from emp\n"
+ + "order by ename limit 5 offset 5";
+ // inputRows = EMP_SIZE = 14
+ // offset + fetch = 5 + 5 = 10
+ // rowBytes = (2 real columns + 3 virtual columns) * 4 bytes per column
+ // = 5 * 4
+ // = 20
+ double cpuCost = Util.nLogM(EMP_SIZE, 10) * 5 * 4;
+ sql(sql).assertCpuCost(is(cpuCost), "offset + fetch smaller than table size "
+ + "=> cpu cost should be: inputRows * log(offset + fetch) * rowBytes");
+ }
+
+ @Test void testSortCpuCostLimit() {
+ final String sql = "select ename, deptno from emp limit 10";
+ final double cpuCost = 10 * 5 * 4;
+ sql(sql).assertCpuCost(is(cpuCost), "no order by clause "
+ + "=> cpu cost should be min(fetch + offset, inputRows) * rowBytes");
+ }
+
+ @Test void testSortCpuCostOffset() {
+ final String sql = "select ename from emp order by ename offset 10";
+ double cpuCost = Util.nLogM(EMP_SIZE, EMP_SIZE) * 4 * 4;
+ sql(sql).assertCpuCost(is(cpuCost), "offset smaller than table size "
+ + "=> cpu cost should be: inputRows * log(inputRows) * rowBytes");
+ }
+
+ @Test void testSortCpuCostLargeOffset() {
+ final String sql = "select ename from emp order by ename offset 100";
+ double cpuCost = Util.nLogM(EMP_SIZE, EMP_SIZE) * 4 * 4;
+ sql(sql).assertCpuCost(is(cpuCost), "offset larger than table size "
+ + "=> cpu cost should be: inputRows * log(inputRows) * rowBytes");
+ }
+
+ @Test void testSortCpuCostLimit0() {
+ final String sql = "select ename from emp order by ename limit 0";
+ sql(sql).assertCpuCost(is(0d), "fetch zero => cpu cost should be 0");
+ }
+
+ @Test void testSortCpuCostLimit1() {
+ final String sql = "select ename, deptno from emp\n"
+ + "order by ename limit 1";
+ double cpuCost = EMP_SIZE * 5 * 4;
+ sql(sql).assertCpuCost(is(cpuCost), "fetch 1 "
+ + "=> cpu cost should be inputRows * rowBytes");
+ }
+
+ @Test void testSortCpuCostLargeLimit() {
+ final String sql = "select ename, deptno from emp\n"
+ + "order by ename limit 10000";
+ double cpuCost = Util.nLogM(EMP_SIZE, EMP_SIZE) * 5 * 4;
+ sql(sql).assertCpuCost(is(cpuCost), "sort limit exceeds table size "
+ + "=> cpu cost should be dominated by table size");
+ }
+
+ // ----------------------------------------------------------------------
+ // Tests for getSelectivity
+ // ----------------------------------------------------------------------
+
private void checkFilterSelectivity(
String sql,
double expected) {
@@ -3271,6 +3369,8 @@ public class RelMetadataTest extends SqlToRelTestBase {
});
}
+ //~ Inner classes and interfaces -------------------------------------------
+
/** Custom metadata interface. */
public interface ColType extends Metadata {
Method METHOD = Types.lookupMethod(ColType.class, "getColType", int.class);
@@ -3356,8 +3456,7 @@ public class RelMetadataTest extends SqlToRelTestBase {
/**
* Dummy rel node used for testing.
*/
- private class DummyRelNode extends SingleRel {
-
+ private static class DummyRelNode extends SingleRel {
/**
* Creates a <code>DummyRelNode</code>.
*/
@@ -3369,8 +3468,8 @@ public class RelMetadataTest extends SqlToRelTestBase {
/**
* Mocked catalog reader for registering table with composite keys.
*/
- private class CompositeKeysCatalogReader extends MockCatalogReaderSimple {
-
+ private static class CompositeKeysCatalogReader
+ extends MockCatalogReaderSimple {
CompositeKeysCatalogReader(RelDataTypeFactory typeFactory, boolean caseSensitive) {
super(typeFactory, caseSensitive);
}
@@ -3390,26 +3489,30 @@ public class RelMetadataTest extends SqlToRelTestBase {
}
}
- /** Test case for
- * <a href="https://issues.apache.org/jira/browse/CALCITE-4192">[CALCITE-4192]
- * RelMdColumnOrigins get the wrong index of group by columns after RelNode was optimized by
- * AggregateProjectMergeRule rule</a>. */
- @Test void testColumnOriginAfterAggProjectMergeRule() {
- final String sql = "select count(ename), SAL from emp group by SAL";
- final RelNode rel = tester.convertSqlToRel(sql).rel;
- final HepProgramBuilder programBuilder = HepProgram.builder();
- programBuilder.addRuleInstance(CoreRules.AGGREGATE_PROJECT_MERGE);
- final HepPlanner planner = new HepPlanner(programBuilder.build());
- planner.setRoot(rel);
- final RelNode optimizedRel = planner.findBestExp();
+ /** Parameters for a test. */
+ private static class Sql {
+ private final Tester tester;
+ private final String sql;
- Set<RelColumnOrigin> origins = RelMetadataQuery.instance()
- .getColumnOrigins(optimizedRel, 1);
- assertThat(origins.size(), equalTo(1));
+ Sql(Tester tester, String sql) {
+ this.tester = tester;
+ this.sql = sql;
+ }
- RelColumnOrigin columnOrigin = origins.iterator().next();
- assertThat(columnOrigin.getOriginColumnOrdinal(), equalTo(5));
- assertThat(columnOrigin.getOriginTable().getRowType().getFieldNames().get(5),
- equalTo("SAL"));
+ Sql assertCpuCost(Matcher<Double> matcher, String reason) {
+ RelNode rel = convertSql(tester, sql);
+ RelOptCost cost = computeRelSelfCost(rel);
+ assertThat(reason + "\n"
+ + "sql:" + sql + "\n"
+ + "plan:" + RelOptUtil.toString(rel, SqlExplainLevel.ALL_ATTRIBUTES),
+ cost.getCpu(), matcher);
+ return this;
+ }
+
+ private static RelOptCost computeRelSelfCost(RelNode rel) {
+ final RelMetadataQuery mq = rel.getCluster().getMetadataQuery();
+ RelOptPlanner planner = new VolcanoPlanner();
+ return rel.computeSelfCost(planner, mq);
+ }
}
}
diff --git a/plus/src/test/java/org/apache/calcite/adapter/tpcds/TpcdsTest.java b/plus/src/test/java/org/apache/calcite/adapter/tpcds/TpcdsTest.java
index 188d589..5d04be4 100644
--- a/plus/src/test/java/org/apache/calcite/adapter/tpcds/TpcdsTest.java
+++ b/plus/src/test/java/org/apache/calcite/adapter/tpcds/TpcdsTest.java
@@ -220,9 +220,9 @@ class TpcdsTest {
.withHook(Hook.PROGRAM, handler(true, 2))
.explainMatches("including all attributes ",
CalciteAssert.checkMaskedResultContains(""
- + "EnumerableCalc(expr#0..9=[{inputs}], expr#10=[/($t4, $t3)], expr#11=[CAST($t10):INTEGER NOT NULL], expr#12=[*($t4, $t4)], expr#13=[/($t12, $t3)], expr#14=[-($t5, $t13)], expr#15=[1], expr#16=[=($t3, $t15)], expr#17=[null:BIGINT], expr#18=[-($t3, $t15)], expr#19=[CASE($t16, $t17, $t18)], expr#20=[/($t14, $t19)], expr#21=[0.5:DECIMAL(2, 1)], expr#22=[POWER($t20, $t21)], expr#23=[CAST($t22):INTEGER NOT NULL], expr#24=[/($t23, $t11)], expr#25=[/($t6, $t3)], expr#26=[CAST($ [...]
- + " EnumerableLimit(fetch=[100]): rowcount = 100.0, cumulative cost = {1.2435775409784036E28 rows, 2.555295485909236E30 cpu, 0.0 io}\n"
- + " EnumerableSort(sort0=[$0], sort1=[$1], sort2=[$2], dir0=[ASC], dir1=[ASC], dir2=[ASC]): rowcount = 5.434029018852197E26, cumulative cost = {1.2435775409784036E28 rows, 2.555295485909236E30 cpu, 0.0 io}\n"
+ + "EnumerableCalc(expr#0..9=[{inputs}], expr#10=[/($t4, $t3)], expr#11=[CAST($t10):INTEGER NOT NULL], expr#12=[*($t4, $t4)], expr#13=[/($t12, $t3)], expr#14=[-($t5, $t13)], expr#15=[1], expr#16=[=($t3, $t15)], expr#17=[null:BIGINT], expr#18=[-($t3, $t15)], expr#19=[CASE($t16, $t17, $t18)], expr#20=[/($t14, $t19)], expr#21=[0.5:DECIMAL(2, 1)], expr#22=[POWER($t20, $t21)], expr#23=[CAST($t22):INTEGER NOT NULL], expr#24=[/($t23, $t11)], expr#25=[/($t6, $t3)], expr#26=[CAST($ [...]
+ + " EnumerableLimit(fetch=[100]): rowcount = 100.0, cumulative cost = {1.2435775409784036E28 rows, 2.95671738161514E30 cpu, 0.0 io}\n"
+ + " EnumerableSort(sort0=[$0], sort1=[$1], sort2=[$2], dir0=[ASC], dir1=[ASC], dir2=[ASC]): rowcount = 5.434029018852197E26, cumulative cost = {1.2435775409784036E28 rows, 2.95671738161514E30 cpu, 0.0 io}\n"
+ " EnumerableAggregate(group=[{0, 1, 2}], STORE_SALES_QUANTITYCOUNT=[COUNT()], agg#1=[$SUM0($3)], agg#2=[$SUM0($6)], agg#3=[$SUM0($4)], agg#4=[$SUM0($7)], agg#5=[$SUM0($5)], agg#6=[$SUM0($8)]): rowcount = 5.434029018852197E26, cumulative cost = {1.1892372507898816E28 rows, 1.2172225002228922E30 cpu, 0.0 io}\n"
+ " EnumerableCalc(expr#0..211=[{inputs}], expr#212=[*($t89, $t89)], expr#213=[*($t140, $t140)], expr#214=[*($t196, $t196)], I_ITEM_ID=[$t58], I_ITEM_DESC=[$t61], S_STATE=[$t24], SS_QUANTITY=[$t89], SR_RETURN_QUANTITY=[$t140], CS_QUANTITY=[$t196], $f6=[$t212], $f7=[$t213], $f8=[$t214]): rowcount = 5.434029018852197E27, cumulative cost = {1.0873492066864028E28 rows, 1.2172225002228922E30 cpu, 0.0 io}\n"
+ " EnumerableHashJoin(condition=[AND(=($82, $133), =($81, $132), =($88, $139))], joinType=[inner]): rowcount = 5.434029018852197E27, cumulative cost = {5.439463048011832E27 rows, 1.7776306E7 cpu, 0.0 io}\n"