You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@calcite.apache.org by hy...@apache.org on 2019/12/03 22:17:02 UTC
[calcite] branch master updated: [CALCITE-3473] Getting unique
result for table scan should contain key column(s) (Wang Yanlin)
This is an automated email from the ASF dual-hosted git repository.
hyuan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/calcite.git
The following commit(s) were added to refs/heads/master by this push:
new ed181a4 [CALCITE-3473] Getting unique result for table scan should contain key column(s) (Wang Yanlin)
ed181a4 is described below
commit ed181a4671b9941f2852c581daa043d8a2d6dec2
Author: yanlin-Lynn <19...@163.com>
AuthorDate: Tue Oct 22 18:58:24 2019 +0800
[CALCITE-3473] Getting unique result for table scan should contain key column(s) (Wang Yanlin)
For table with key column(s) defined, we can expect uniqueness on key column(s),
so the result should contain key column(s).
Close #1557
---
.../apache/calcite/plan/RelOptAbstractTable.java | 5 +
.../java/org/apache/calcite/plan/RelOptTable.java | 6 +
.../apache/calcite/prepare/RelOptTableImpl.java | 4 +
.../calcite/rel/metadata/RelMdUniqueKeys.java | 10 ++
.../java/org/apache/calcite/schema/Statistic.java | 3 +
.../java/org/apache/calcite/schema/Statistics.java | 8 ++
.../rel/rel2sql/RelToSqlConverterStructsTest.java | 4 +
.../org/apache/calcite/test/RelMetadataTest.java | 143 +++++++++++++++------
.../org/apache/calcite/test/SqlToRelTestBase.java | 8 ++
.../calcite/test/catalog/MockCatalogReader.java | 11 ++
10 files changed, 166 insertions(+), 36 deletions(-)
diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptAbstractTable.java b/core/src/main/java/org/apache/calcite/plan/RelOptAbstractTable.java
index 854e42f..18e03ee 100644
--- a/core/src/main/java/org/apache/calcite/plan/RelOptAbstractTable.java
+++ b/core/src/main/java/org/apache/calcite/plan/RelOptAbstractTable.java
@@ -97,6 +97,11 @@ public abstract class RelOptAbstractTable implements RelOptTable {
return false;
}
+ // Override to get unique keys
+ public List<ImmutableBitSet> getKeys() {
+ return Collections.emptyList();
+ }
+
// Override to define foreign keys
public List<RelReferentialConstraint> getReferentialConstraints() {
return Collections.emptyList();
diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptTable.java b/core/src/main/java/org/apache/calcite/plan/RelOptTable.java
index 1e351ac..dca3ab2 100644
--- a/core/src/main/java/org/apache/calcite/plan/RelOptTable.java
+++ b/core/src/main/java/org/apache/calcite/plan/RelOptTable.java
@@ -100,6 +100,12 @@ public interface RelOptTable extends Wrapper {
boolean isKey(ImmutableBitSet columns);
/**
+ * Returns a list of unique keys, empty list if no key exist,
+ * the result should be consistent with {@code isKey}
+ */
+ List<ImmutableBitSet> getKeys();
+
+ /**
* Returns the referential constraints existing for this table. These constraints
* are represented over other tables using {@link RelReferentialConstraint} nodes.
*/
diff --git a/core/src/main/java/org/apache/calcite/prepare/RelOptTableImpl.java b/core/src/main/java/org/apache/calcite/prepare/RelOptTableImpl.java
index 62f4dfe..c3706d3 100644
--- a/core/src/main/java/org/apache/calcite/prepare/RelOptTableImpl.java
+++ b/core/src/main/java/org/apache/calcite/prepare/RelOptTableImpl.java
@@ -319,6 +319,10 @@ public class RelOptTableImpl extends Prepare.AbstractPreparingTable {
return false;
}
+ public List<ImmutableBitSet> getKeys() {
+ return table.getStatistic().getKeys();
+ }
+
public List<RelReferentialConstraint> getReferentialConstraints() {
if (table != null) {
return table.getStatistic().getReferentialConstraints();
diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUniqueKeys.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUniqueKeys.java
index 9ced406..a116257 100644
--- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUniqueKeys.java
+++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUniqueKeys.java
@@ -26,6 +26,7 @@ import org.apache.calcite.rel.core.Project;
import org.apache.calcite.rel.core.SetOp;
import org.apache.calcite.rel.core.Sort;
import org.apache.calcite.rel.core.TableModify;
+import org.apache.calcite.rel.core.TableScan;
import org.apache.calcite.rex.RexInputRef;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.util.BuiltInMethod;
@@ -226,6 +227,15 @@ public class RelMdUniqueKeys
return ImmutableSet.of();
}
+ public Set<ImmutableBitSet> getUniqueKeys(TableScan rel, RelMetadataQuery mq,
+ boolean ignoreNulls) {
+ final List<ImmutableBitSet> keys = rel.getTable().getKeys();
+ for (ImmutableBitSet key : keys) {
+ assert rel.getTable().isKey(key);
+ }
+ return ImmutableSet.copyOf(keys);
+ }
+
// Catch-all rule when none of the others apply.
public Set<ImmutableBitSet> getUniqueKeys(RelNode rel, RelMetadataQuery mq,
boolean ignoreNulls) {
diff --git a/core/src/main/java/org/apache/calcite/schema/Statistic.java b/core/src/main/java/org/apache/calcite/schema/Statistic.java
index 4b53d89..0d1fdc6 100644
--- a/core/src/main/java/org/apache/calcite/schema/Statistic.java
+++ b/core/src/main/java/org/apache/calcite/schema/Statistic.java
@@ -39,6 +39,9 @@ public interface Statistic {
*/
boolean isKey(ImmutableBitSet columns);
+ /** Returns a list of unique keys, or null if no key exist. */
+ List<ImmutableBitSet> getKeys();
+
/** Returns the collection of referential constraints (foreign-keys)
* for this table. */
List<RelReferentialConstraint> getReferentialConstraints();
diff --git a/core/src/main/java/org/apache/calcite/schema/Statistics.java b/core/src/main/java/org/apache/calcite/schema/Statistics.java
index 0e55140..7f5ef5a 100644
--- a/core/src/main/java/org/apache/calcite/schema/Statistics.java
+++ b/core/src/main/java/org/apache/calcite/schema/Statistics.java
@@ -44,6 +44,10 @@ public class Statistics {
return false;
}
+ public List<ImmutableBitSet> getKeys() {
+ return ImmutableList.of();
+ }
+
public List<RelReferentialConstraint> getReferentialConstraints() {
return ImmutableList.of();
}
@@ -98,6 +102,10 @@ public class Statistics {
return false;
}
+ public List<ImmutableBitSet> getKeys() {
+ return ImmutableList.copyOf(keys);
+ }
+
public List<RelReferentialConstraint> getReferentialConstraints() {
return referentialConstraints;
}
diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterStructsTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterStructsTest.java
index d6aac9a..fd97089 100644
--- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterStructsTest.java
+++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterStructsTest.java
@@ -157,6 +157,10 @@ public class RelToSqlConverterStructsTest {
return false;
}
+ @Override public List<ImmutableBitSet> getKeys() {
+ return ImmutableList.of();
+ }
+
@Override public List<RelReferentialConstraint> getReferentialConstraints() {
return ImmutableList.of();
}
diff --git a/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java b/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java
index 739ab2d..7972717 100644
--- a/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java
+++ b/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java
@@ -73,6 +73,7 @@ import org.apache.calcite.rel.metadata.RelMdUtil;
import org.apache.calcite.rel.metadata.RelMetadataProvider;
import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexCorrelVariable;
@@ -85,8 +86,11 @@ import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.SqlOperator;
import org.apache.calcite.sql.SqlSpecialOperator;
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.test.SqlTestFactory;
import org.apache.calcite.sql.type.ReturnTypes;
import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.test.catalog.MockCatalogReader;
+import org.apache.calcite.test.catalog.MockCatalogReaderSimple;
import org.apache.calcite.tools.FrameworkConfig;
import org.apache.calcite.tools.Frameworks;
import org.apache.calcite.tools.RelBuilder;
@@ -182,6 +186,10 @@ public class RelMetadataTest extends SqlToRelTestBase {
// ----------------------------------------------------------------------
private RelNode convertSql(String sql) {
+ return convertSql(tester, sql);
+ }
+
+ private RelNode convertSql(Tester tester, String sql) {
final RelRoot root = tester.convertSqlToRel(sql);
root.rel.getCluster().setMetadataProvider(DefaultRelMetadataProvider.INSTANCE);
return root.rel;
@@ -874,6 +882,33 @@ public class RelMetadataTest extends SqlToRelTestBase {
assertThat(result, is(1D));
}
+ // ----------------------------------------------------------------------
+ // Tests for getUniqueKeys
+ // ----------------------------------------------------------------------
+
+ /**
+ * Checks result of getting unique keys for sql, using specific tester.
+ */
+ private void checkGetUniqueKeys(Tester tester,
+ String sql, Set<ImmutableBitSet> expectedUniqueKeySet) {
+ RelNode rel = convertSql(tester, sql);
+ final RelMetadataQuery mq = rel.getCluster().getMetadataQuery();
+ Set<ImmutableBitSet> result = mq.getUniqueKeys(rel);
+ assertThat(result, CoreMatchers.equalTo(expectedUniqueKeySet));
+ assertUniqueConsistent(rel);
+ }
+
+ /**
+ * Checks result of getting unique keys for sql, using default tester.
+ */
+ private void checkGetUniqueKeys(String sql, Set<ImmutableBitSet> expectedUniqueKeySet) {
+ RelNode rel = convertSql(sql);
+ final RelMetadataQuery mq = rel.getCluster().getMetadataQuery();
+ Set<ImmutableBitSet> result = mq.getUniqueKeys(rel);
+ assertThat(result, CoreMatchers.equalTo(expectedUniqueKeySet));
+ assertUniqueConsistent(rel);
+ }
+
/** Asserts that {@link RelMetadataQuery#getUniqueKeys(RelNode)}
* and {@link RelMetadataQuery#areColumnsUnique(RelNode, ImmutableBitSet)}
* return consistent results. */
@@ -904,11 +939,8 @@ public class RelMetadataTest extends SqlToRelTestBase {
* "RelMdColumnUniqueness uses ImmutableBitSet.Builder twice, gets
* NullPointerException"</a>. */
@Test public void testJoinUniqueKeys() {
- RelNode rel = convertSql("select * from emp join bonus using (ename)");
- final RelMetadataQuery mq = rel.getCluster().getMetadataQuery();
- Set<ImmutableBitSet> result = mq.getUniqueKeys(rel);
- assertThat(result.isEmpty(), is(true));
- assertUniqueConsistent(rel);
+ checkGetUniqueKeys("select * from emp join bonus using (ename)",
+ ImmutableSet.of());
}
@Test public void testCorrelateUniqueKeys() {
@@ -937,23 +969,13 @@ public class RelMetadataTest extends SqlToRelTestBase {
}
@Test public void testGroupByEmptyUniqueKeys() {
- RelNode rel = convertSql("select count(*) from emp");
- final RelMetadataQuery mq = rel.getCluster().getMetadataQuery();
- Set<ImmutableBitSet> result = mq.getUniqueKeys(rel);
- assertThat(result,
- CoreMatchers.equalTo(
- ImmutableSet.of(ImmutableBitSet.of())));
- assertUniqueConsistent(rel);
+ checkGetUniqueKeys("select count(*) from emp",
+ ImmutableSet.of(ImmutableBitSet.of()));
}
@Test public void testGroupByEmptyHavingUniqueKeys() {
- RelNode rel = convertSql("select count(*) from emp where 1 = 1");
- final RelMetadataQuery mq = rel.getCluster().getMetadataQuery();
- final Set<ImmutableBitSet> result = mq.getUniqueKeys(rel);
- assertThat(result,
- CoreMatchers.equalTo(
- ImmutableSet.of(ImmutableBitSet.of())));
- assertUniqueConsistent(rel);
+ checkGetUniqueKeys("select count(*) from emp where 1 = 1",
+ ImmutableSet.of(ImmutableBitSet.of()));
}
@Test public void testFullOuterJoinUniqueness1() {
@@ -1111,26 +1133,51 @@ public class RelMetadataTest extends SqlToRelTestBase {
}
@Test public void testGroupBy() {
- RelNode rel = convertSql("select deptno, count(*), sum(sal) from emp\n"
- + "group by deptno");
- final RelMetadataQuery mq = rel.getCluster().getMetadataQuery();
- final Set<ImmutableBitSet> result = mq.getUniqueKeys(rel);
- assertThat(result,
- CoreMatchers.equalTo(
- ImmutableSet.of(ImmutableBitSet.of(0))));
- assertUniqueConsistent(rel);
+ checkGetUniqueKeys("select deptno, count(*), sum(sal) from emp group by deptno",
+ ImmutableSet.of(ImmutableBitSet.of(0)));
}
@Test public void testUnion() {
- RelNode rel = convertSql("select deptno from emp\n"
- + "union\n"
- + "select deptno from dept");
- final RelMetadataQuery mq = rel.getCluster().getMetadataQuery();
- final Set<ImmutableBitSet> result = mq.getUniqueKeys(rel);
- assertThat(result,
- CoreMatchers.equalTo(
- ImmutableSet.of(ImmutableBitSet.of(0))));
- assertUniqueConsistent(rel);
+ checkGetUniqueKeys("select deptno from emp\n"
+ + "union\n"
+ + "select deptno from dept",
+ ImmutableSet.of(ImmutableBitSet.of(0)));
+ }
+
+ @Test public void testSingleKeyTableScanUniqueKeys() {
+ // select key column
+ checkGetUniqueKeys("select empno, ename from emp",
+ ImmutableSet.of(ImmutableBitSet.of(0)));
+
+ // select non key column
+ checkGetUniqueKeys("select ename, deptno from emp",
+ ImmutableSet.of());
+ }
+
+ @Test public void testCompositeKeysTableScanUniqueKeys() {
+ SqlTestFactory.MockCatalogReaderFactory factory = (typeFactory, caseSensitive) -> {
+ CompositeKeysCatalogReader catalogReader =
+ new CompositeKeysCatalogReader(typeFactory, false);
+ catalogReader.init();
+ return catalogReader;
+ };
+ Tester newTester = tester.withCatalogReaderFactory(factory);
+
+ // all columns, contain composite keys
+ checkGetUniqueKeys(newTester, "select * from s.composite_keys_table",
+ ImmutableSet.of(ImmutableBitSet.of(0, 1)));
+
+ // only contain composite keys
+ checkGetUniqueKeys(newTester, "select key1, key2 from s.composite_keys_table",
+ ImmutableSet.of(ImmutableBitSet.of(0, 1)));
+
+ // partial column of composite keys
+ checkGetUniqueKeys(newTester, "select key1, value1 from s.composite_keys_table",
+ ImmutableSet.of());
+
+ // no column of composite keys
+ checkGetUniqueKeys(newTester, "select value1 from s.composite_keys_table",
+ ImmutableSet.of());
}
@Test public void testBrokenCustomProviderWithMetadataFactory() {
@@ -3005,6 +3052,30 @@ public class RelMetadataTest extends SqlToRelTestBase {
super(cluster, traits, input);
}
}
+
+ /**
+ * Mocked catalog reader for registering table with composite keys.
+ */
+ private class CompositeKeysCatalogReader extends MockCatalogReaderSimple {
+
+ CompositeKeysCatalogReader(RelDataTypeFactory typeFactory, boolean caseSensitive) {
+ super(typeFactory, caseSensitive);
+ }
+
+ @Override public MockCatalogReader init() {
+ super.init();
+ MockSchema tSchema = new MockSchema("s");
+ registerSchema(tSchema);
+ // Register "T1" table.
+ final MockTable t1 =
+ MockTable.create(this, tSchema, "composite_keys_table", false, 7.0, null);
+ t1.addColumn("key1", typeFactory.createSqlType(SqlTypeName.VARCHAR), true);
+ t1.addColumn("key2", typeFactory.createSqlType(SqlTypeName.VARCHAR), true);
+ t1.addColumn("value1", typeFactory.createSqlType(SqlTypeName.INTEGER));
+ registerTable(t1);
+ return this;
+ }
+ }
}
// End RelMetadataTest.java
diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelTestBase.java b/core/src/test/java/org/apache/calcite/test/SqlToRelTestBase.java
index 436a52c..ae09d2f 100644
--- a/core/src/test/java/org/apache/calcite/test/SqlToRelTestBase.java
+++ b/core/src/test/java/org/apache/calcite/test/SqlToRelTestBase.java
@@ -432,6 +432,10 @@ public abstract class SqlToRelTestBase {
return false;
}
+ public List<ImmutableBitSet> getKeys() {
+ return ImmutableList.of();
+ }
+
public List<RelReferentialConstraint> getReferentialConstraints() {
return ImmutableList.of();
}
@@ -510,6 +514,10 @@ public abstract class SqlToRelTestBase {
return parent.isKey(columns);
}
+ public List<ImmutableBitSet> getKeys() {
+ return parent.getKeys();
+ }
+
public List<RelReferentialConstraint> getReferentialConstraints() {
return parent.getReferentialConstraints();
}
diff --git a/core/src/test/java/org/apache/calcite/test/catalog/MockCatalogReader.java b/core/src/test/java/org/apache/calcite/test/catalog/MockCatalogReader.java
index 97179b0..590c5a5 100644
--- a/core/src/test/java/org/apache/calcite/test/catalog/MockCatalogReader.java
+++ b/core/src/test/java/org/apache/calcite/test/catalog/MockCatalogReader.java
@@ -515,6 +515,13 @@ public abstract class MockCatalogReader extends CalciteCatalogReader {
&& columns.contains(ImmutableBitSet.of(keyList));
}
+ public List<ImmutableBitSet> getKeys() {
+ if (keyList.isEmpty()) {
+ return ImmutableList.of();
+ }
+ return ImmutableList.of(ImmutableBitSet.of(keyList));
+ }
+
public List<RelReferentialConstraint> getReferentialConstraints() {
return referentialConstraints;
}
@@ -1052,6 +1059,10 @@ public abstract class MockCatalogReader extends CalciteCatalogReader {
return table.isKey(columns);
}
+ public List<ImmutableBitSet> getKeys() {
+ return table.getKeys();
+ }
+
public List<RelReferentialConstraint> getReferentialConstraints() {
return table.getReferentialConstraints();
}