You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@calcite.apache.org by jh...@apache.org on 2017/11/29 04:09:01 UTC
[2/5] calcite git commit: [CALCITE-1616] Data profiler
http://git-wip-us.apache.org/repos/asf/calcite/blob/dad58186/core/src/main/java/org/apache/calcite/util/PartiallyOrderedSet.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/util/PartiallyOrderedSet.java b/core/src/main/java/org/apache/calcite/util/PartiallyOrderedSet.java
index ad808a9..a3db6db 100644
--- a/core/src/main/java/org/apache/calcite/util/PartiallyOrderedSet.java
+++ b/core/src/main/java/org/apache/calcite/util/PartiallyOrderedSet.java
@@ -16,7 +16,11 @@
*/
package org.apache.calcite.util;
-import java.util.AbstractList;
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+
import java.util.AbstractSet;
import java.util.ArrayDeque;
import java.util.ArrayList;
@@ -61,7 +65,21 @@ import java.util.Set;
* @param <E> Element type
*/
public class PartiallyOrderedSet<E> extends AbstractSet<E> {
+ /** Ordering that orders bit sets by inclusion.
+ *
+ * <p>For example, the children of 14 (1110) are 12 (1100), 10 (1010) and
+ * 6 (0110).
+ */
+ public static final Ordering<ImmutableBitSet> BIT_SET_INCLUSION_ORDERING =
+ new Ordering<ImmutableBitSet>() {
+ public boolean lessThan(ImmutableBitSet e1, ImmutableBitSet e2) {
+ return e1.contains(e2);
+ }
+ };
+
private final Map<E, Node<E>> map;
+ private final Function<E, Iterable<E>> parentFunction;
+ private final Function<E, Iterable<E>> childFunction;
private final Ordering<E> ordering;
/**
@@ -71,7 +89,9 @@ public class PartiallyOrderedSet<E> extends AbstractSet<E> {
private final Node<E> topNode;
private final Node<E> bottomNode;
- private static final boolean DEBUG = Math.random() >= 0;
+ /** Whether to check internal consistency all the time.
+ * False unless you specify "-Dcalcite.debug" on the command line. */
+ private static final boolean DEBUG = Util.getBooleanProperty("calcite.debug");
/**
* Creates a partially-ordered set.
@@ -79,7 +99,19 @@ public class PartiallyOrderedSet<E> extends AbstractSet<E> {
* @param ordering Ordering relation
*/
public PartiallyOrderedSet(Ordering<E> ordering) {
- this(ordering, new HashMap<E, Node<E>>());
+ this(ordering, new HashMap<E, Node<E>>(), null, null);
+ }
+
+ /**
+ * Creates a partially-ordered set with a parent-generating function.
+ *
+ * @param ordering Ordering relation
+ * @param parentFunction Function to compute parents of a node; may be null
+ */
+ public PartiallyOrderedSet(Ordering<E> ordering,
+ Function<E, Iterable<E>> childFunction,
+ Function<E, Iterable<E>> parentFunction) {
+ this(ordering, new HashMap<E, Node<E>>(), childFunction, parentFunction);
}
/**
@@ -90,7 +122,8 @@ public class PartiallyOrderedSet<E> extends AbstractSet<E> {
* @param collection Initial contents of partially-ordered set
*/
public PartiallyOrderedSet(Ordering<E> ordering, Collection<E> collection) {
- this(ordering, new HashMap<E, Node<E>>(collection.size() * 3 / 2));
+ this(ordering, new HashMap<E, Node<E>>(collection.size() * 3 / 2), null,
+ null);
addAll(collection);
}
@@ -99,10 +132,15 @@ public class PartiallyOrderedSet<E> extends AbstractSet<E> {
*
* @param ordering Ordering relation
* @param map Map from values to nodes
+ * @param parentFunction Function to compute parents of a node; may be null
*/
- private PartiallyOrderedSet(Ordering<E> ordering, Map<E, Node<E>> map) {
+ private PartiallyOrderedSet(Ordering<E> ordering, Map<E, Node<E>> map,
+ Function<E, Iterable<E>> childFunction,
+ Function<E, Iterable<E>> parentFunction) {
this.ordering = ordering;
this.map = map;
+ this.childFunction = childFunction;
+ this.parentFunction = parentFunction;
this.topNode = new TopBottomNode<>(true);
this.bottomNode = new TopBottomNode<>(false);
this.topNode.childList.add(bottomNode);
@@ -528,7 +566,7 @@ public class PartiallyOrderedSet<E> extends AbstractSet<E> {
* Returns the values in this partially-ordered set that are less-than
* a given value and there are no intervening values.
*
- * <p>If the value is not in this set, returns the empty list.</p>
+ * <p>If the value is not in this set, returns null.
*
* @see #getDescendants
*
@@ -537,15 +575,33 @@ public class PartiallyOrderedSet<E> extends AbstractSet<E> {
* value
*/
public List<E> getChildren(E e) {
+ return getChildren(e, false);
+ }
+
+ /**
+ * Returns the values in this partially-ordered set that are less-than
+ * a given value and there are no intervening values.
+ *
+ * <p>If the value is not in this set, returns null if {@code hypothetical}
+ * is false.
+ *
+ * @see #getDescendants
+ *
+ * @param e Value
+ * @param hypothetical Whether to generate a list if value is not in the set
+ * @return List of values in this set that are directly less than the given
+ * value
+ */
+ public List<E> getChildren(E e, boolean hypothetical) {
final Node<E> node = map.get(e);
if (node == null) {
- return null;
- } else if (node.childList.get(0).e == null) {
- // child list contains bottom element, so officially there are no
- // children
- return Collections.emptyList();
+ if (hypothetical) {
+ return strip(findChildren(e));
+ } else {
+ return null;
+ }
} else {
- return new StripList<>(node.childList);
+ return strip(node.childList);
}
}
@@ -553,7 +609,7 @@ public class PartiallyOrderedSet<E> extends AbstractSet<E> {
* Returns the values in this partially-ordered set that are greater-than
* a given value and there are no intervening values.
*
- * <p>If the value is not in this set, returns the empty list.</p>
+ * <p>If the value is not in this set, returns null.
*
* @see #getAncestors
*
@@ -562,32 +618,61 @@ public class PartiallyOrderedSet<E> extends AbstractSet<E> {
* given value
*/
public List<E> getParents(E e) {
+ return getParents(e, false);
+ }
+
+ /**
+ * Returns the values in this partially-ordered set that are greater-than
+ * a given value and there are no intervening values.
+ *
+ * <p>If the value is not in this set, returns {@code null} if
+ * {@code hypothetical} is false.
+ *
+ * @see #getAncestors
+ *
+ * @param e Value
+ * @param hypothetical Whether to generate a list if value is not in the set
+ * @return List of values in this set that are directly greater than the
+ * given value
+ */
+ public List<E> getParents(E e, boolean hypothetical) {
final Node<E> node = map.get(e);
if (node == null) {
- return null;
- } else if (node.parentList.get(0).e == null) {
- // parent list contains top element, so officially there are no
- // parents
- return Collections.emptyList();
+ if (hypothetical) {
+ if (parentFunction != null) {
+ final List<E> list = new ArrayList<>();
+ closure(parentFunction, e, list, new HashSet<E>());
+ return list;
+ } else {
+ return ImmutableList.copyOf(strip(findParents(e)));
+ }
+ } else {
+ return null;
+ }
} else {
- return new StripList<>(node.parentList);
+ return strip(node.parentList);
}
}
- public List<E> getNonChildren() {
- if (topNode.childList.size() == 1
- && topNode.childList.get(0).e == null) {
- return Collections.emptyList();
+ private void closure(Function<E, Iterable<E>> generator, E e, List<E> list,
+ Set<E> set) {
+ for (E p : Preconditions.checkNotNull(generator.apply(e))) {
+ if (set.add(e)) {
+ if (map.containsKey(p)) {
+ list.add(p);
+ } else {
+ closure(generator, p, list, set);
+ }
+ }
}
- return new StripList<>(topNode.childList);
+ }
+
+ public List<E> getNonChildren() {
+ return strip(topNode.childList);
}
public List<E> getNonParents() {
- if (bottomNode.parentList.size() == 1
- && bottomNode.parentList.get(0).e == null) {
- return Collections.emptyList();
- }
- return new StripList<>(bottomNode.parentList);
+ return strip(bottomNode.parentList);
}
@Override public void clear() {
@@ -612,6 +697,57 @@ public class PartiallyOrderedSet<E> extends AbstractSet<E> {
return descendants(e, true);
}
+ /** Returns a list, backed by a list of
+ * {@link org.apache.calcite.util.PartiallyOrderedSet.Node}s, that strips
+ * away the node and returns the element inside.
+ *
+ * @param <E> Element type
+ */
+ public static <E> List<E> strip(List<Node<E>> list) {
+ if (list.size() == 1
+ && list.get(0).e == null) {
+ // If parent list contains top element, a node whose element is null,
+ // officially there are no parents.
+ // Similarly child list and bottom element.
+ return ImmutableList.of();
+ }
+ return Lists.transform(list,
+ new Function<Node<E>, E>() {
+ public E apply(Node<E> node) {
+ return node.e;
+ }
+ });
+ }
+
+ /** Converts an iterable of nodes into the list of the elements inside.
+ * If there is one node whose element is null, it represents a list
+ * containing either the top or bottom element, so we return the empty list.
+ *
+ * @param <E> Element type
+ */
+ private static <E> ImmutableList<E> strip(Iterable<Node<E>> iterable) {
+ final Iterator<Node<E>> iterator = iterable.iterator();
+ if (!iterator.hasNext()) {
+ return ImmutableList.of();
+ }
+ Node<E> node = iterator.next();
+ if (!iterator.hasNext()) {
+ if (node.e == null) {
+ return ImmutableList.of();
+ } else {
+ return ImmutableList.of(node.e);
+ }
+ }
+ final ImmutableList.Builder<E> builder = ImmutableList.builder();
+ for (;;) {
+ builder.add(node.e);
+ if (!iterator.hasNext()) {
+ return builder.build();
+ }
+ node = iterator.next();
+ }
+ }
+
/**
* Returns a list of values in the set that are less-than a given value.
* The list is in topological order but order is otherwise
@@ -725,27 +861,6 @@ public class PartiallyOrderedSet<E> extends AbstractSet<E> {
*/
boolean lessThan(E e1, E e2);
}
-
- /** List, backed by a list of {@link Node}s, that strips away the
- * node and returns the element inside.
- *
- * @param <E> Element type
- */
- private static class StripList<E> extends AbstractList<E> {
- private final List<Node<E>> list;
-
- StripList(List<Node<E>> list) {
- this.list = list;
- }
-
- @Override public E get(int index) {
- return list.get(index).e;
- }
-
- @Override public int size() {
- return list.size();
- }
- }
}
// End PartiallyOrderedSet.java
http://git-wip-us.apache.org/repos/asf/calcite/blob/dad58186/core/src/test/java/org/apache/calcite/profile/ProfilerTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/calcite/profile/ProfilerTest.java b/core/src/test/java/org/apache/calcite/profile/ProfilerTest.java
new file mode 100644
index 0000000..e45db8b
--- /dev/null
+++ b/core/src/test/java/org/apache/calcite/profile/ProfilerTest.java
@@ -0,0 +1,682 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.profile;
+
+import org.apache.calcite.jdbc.CalciteConnection;
+import org.apache.calcite.linq4j.AbstractEnumerable;
+import org.apache.calcite.linq4j.Enumerable;
+import org.apache.calcite.linq4j.Enumerator;
+import org.apache.calcite.rel.metadata.NullSentinel;
+import org.apache.calcite.runtime.PredicateImpl;
+import org.apache.calcite.test.CalciteAssert;
+import org.apache.calcite.test.Matchers;
+import org.apache.calcite.util.ImmutableBitSet;
+import org.apache.calcite.util.JsonBuilder;
+import org.apache.calcite.util.Pair;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.base.Supplier;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Ordering;
+
+import org.hamcrest.Matcher;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertThat;
+
+/**
+ * Unit tests for {@link Profiler}.
+ */
+public class ProfilerTest {
+ @Test public void testProfileZeroRows() throws Exception {
+ final String sql = "select * from \"scott\".dept where false";
+ sql(sql).unordered(
+ "{type:distribution,columns:[DEPTNO,DNAME,LOC],cardinality:0.0}",
+ "{type:distribution,columns:[DEPTNO,DNAME],cardinality:0.0}",
+ "{type:distribution,columns:[DEPTNO,LOC],cardinality:0.0}",
+ "{type:distribution,columns:[DEPTNO],values:[],cardinality:0.0}",
+ "{type:distribution,columns:[DNAME,LOC],cardinality:0.0}",
+ "{type:distribution,columns:[DNAME],values:[],cardinality:0.0}",
+ "{type:distribution,columns:[LOC],values:[],cardinality:0.0}",
+ "{type:distribution,columns:[],cardinality:0.0}",
+ "{type:rowCount,rowCount:0}",
+ "{type:unique,columns:[]}");
+ }
+
+ @Test public void testProfileOneRow() throws Exception {
+ final String sql = "select * from \"scott\".dept where deptno = 10";
+ sql(sql).unordered(
+ "{type:distribution,columns:[DEPTNO,DNAME,LOC],cardinality:1.0}",
+ "{type:distribution,columns:[DEPTNO,DNAME],cardinality:1.0}",
+ "{type:distribution,columns:[DEPTNO,LOC],cardinality:1.0}",
+ "{type:distribution,columns:[DEPTNO],values:[10],cardinality:1.0}",
+ "{type:distribution,columns:[DNAME,LOC],cardinality:1.0}",
+ "{type:distribution,columns:[DNAME],values:[ACCOUNTING],cardinality:1.0}",
+ "{type:distribution,columns:[LOC],values:[NEWYORK],cardinality:1.0}",
+ "{type:distribution,columns:[],cardinality:1.0}",
+ "{type:rowCount,rowCount:1}",
+ "{type:unique,columns:[]}");
+ }
+
+ @Test public void testProfileTwoRows() throws Exception {
+ final String sql = "select * from \"scott\".dept where deptno in (10, 20)";
+ sql(sql).unordered(
+ "{type:distribution,columns:[DEPTNO,DNAME,LOC],cardinality:2.0}",
+ "{type:distribution,columns:[DEPTNO,DNAME],cardinality:2.0}",
+ "{type:distribution,columns:[DEPTNO,LOC],cardinality:2.0}",
+ "{type:distribution,columns:[DEPTNO],values:[10,20],cardinality:2.0}",
+ "{type:distribution,columns:[DNAME,LOC],cardinality:2.0}",
+ "{type:distribution,columns:[DNAME],values:[ACCOUNTING,RESEARCH],cardinality:2.0}",
+ "{type:distribution,columns:[LOC],values:[DALLAS,NEWYORK],cardinality:2.0}",
+ "{type:distribution,columns:[],cardinality:1.0}",
+ "{type:rowCount,rowCount:2}",
+ "{type:unique,columns:[DEPTNO]}",
+ "{type:unique,columns:[DNAME]}",
+ "{type:unique,columns:[LOC]}");
+ }
+
+ @Test public void testProfileScott() throws Exception {
+ final String sql = "select * from \"scott\".emp\n"
+ + "join \"scott\".dept using (deptno)";
+ sql(sql)
+ .where(new PredicateImpl<Profiler.Statistic>() {
+ public boolean test(Profiler.Statistic statistic) {
+ return !(statistic instanceof Profiler.Distribution)
+ || ((Profiler.Distribution) statistic).cardinality < 14
+ && ((Profiler.Distribution) statistic).minimal;
+ }
+ }).unordered(
+ "{type:distribution,columns:[COMM,DEPTNO0],cardinality:5.0}",
+ "{type:distribution,columns:[COMM,DEPTNO],cardinality:5.0}",
+ "{type:distribution,columns:[COMM,DNAME],cardinality:5.0}",
+ "{type:distribution,columns:[COMM,LOC],cardinality:5.0}",
+ "{type:distribution,columns:[COMM],values:[0.00,300.00,500.00,1400.00],cardinality:5.0,nullCount:10}",
+ "{type:distribution,columns:[DEPTNO,DEPTNO0],cardinality:3.0}",
+ "{type:distribution,columns:[DEPTNO,DNAME],cardinality:3.0}",
+ "{type:distribution,columns:[DEPTNO,LOC],cardinality:3.0}",
+ "{type:distribution,columns:[DEPTNO0,DNAME],cardinality:3.0}",
+ "{type:distribution,columns:[DEPTNO0,LOC],cardinality:3.0}",
+ "{type:distribution,columns:[DEPTNO0],values:[10,20,30],cardinality:3.0}",
+ "{type:distribution,columns:[DEPTNO],values:[10,20,30],cardinality:3.0}",
+ "{type:distribution,columns:[DNAME,LOC],cardinality:3.0}",
+ "{type:distribution,columns:[DNAME],values:[ACCOUNTING,RESEARCH,SALES],cardinality:3.0}",
+ "{type:distribution,columns:[HIREDATE,COMM],cardinality:5.0}",
+ "{type:distribution,columns:[HIREDATE],values:[1980-12-17,1981-01-05,1981-02-04,1981-02-20,1981-02-22,1981-06-09,1981-09-08,1981-09-28,1981-11-17,1981-12-03,1982-01-23,1987-04-19,1987-05-23],cardinality:13.0}",
+ "{type:distribution,columns:[JOB,COMM],cardinality:5.0}",
+ "{type:distribution,columns:[JOB,DEPTNO0],cardinality:9.0}",
+ "{type:distribution,columns:[JOB,DEPTNO],cardinality:9.0}",
+ "{type:distribution,columns:[JOB,DNAME],cardinality:9.0}",
+ "{type:distribution,columns:[JOB,LOC],cardinality:9.0}",
+ "{type:distribution,columns:[JOB,MGR,DEPTNO0],cardinality:10.0}",
+ "{type:distribution,columns:[JOB,MGR,DEPTNO],cardinality:10.0}",
+ "{type:distribution,columns:[JOB,MGR,DNAME],cardinality:10.0}",
+ "{type:distribution,columns:[JOB,MGR,LOC],cardinality:10.0}",
+ "{type:distribution,columns:[JOB,MGR],cardinality:8.0}",
+ "{type:distribution,columns:[JOB,SAL],cardinality:12.0}",
+ "{type:distribution,columns:[JOB],values:[ANALYST,CLERK,MANAGER,PRESIDENT,SALESMAN],cardinality:5.0}",
+ "{type:distribution,columns:[LOC],values:[CHICAGO,DALLAS,NEWYORK],cardinality:3.0}",
+ "{type:distribution,columns:[MGR,COMM],cardinality:5.0}",
+ "{type:distribution,columns:[MGR,DEPTNO0],cardinality:9.0}",
+ "{type:distribution,columns:[MGR,DEPTNO],cardinality:9.0}",
+ "{type:distribution,columns:[MGR,DNAME],cardinality:9.0}",
+ "{type:distribution,columns:[MGR,LOC],cardinality:9.0}",
+ "{type:distribution,columns:[MGR,SAL],cardinality:12.0}",
+ "{type:distribution,columns:[MGR],values:[7566,7698,7782,7788,7839,7902],cardinality:7.0,nullCount:1}",
+ "{type:distribution,columns:[SAL,COMM],cardinality:5.0}",
+ "{type:distribution,columns:[SAL,DEPTNO0],cardinality:12.0}",
+ "{type:distribution,columns:[SAL,DEPTNO],cardinality:12.0}",
+ "{type:distribution,columns:[SAL,DNAME],cardinality:12.0}",
+ "{type:distribution,columns:[SAL,LOC],cardinality:12.0}",
+ "{type:distribution,columns:[SAL],values:[800.00,950.00,1100.00,1250.00,1300.00,1500.00,1600.00,2450.00,2850.00,2975.00,3000.00,5000.00],cardinality:12.0}",
+ "{type:distribution,columns:[],cardinality:1.0}",
+ "{type:fd,columns:[DEPTNO0],dependentColumn:DEPTNO}",
+ "{type:fd,columns:[DEPTNO0],dependentColumn:DNAME}",
+ "{type:fd,columns:[DEPTNO0],dependentColumn:LOC}",
+ "{type:fd,columns:[DEPTNO],dependentColumn:DEPTNO0}",
+ "{type:fd,columns:[DEPTNO],dependentColumn:DNAME}",
+ "{type:fd,columns:[DEPTNO],dependentColumn:LOC}",
+ "{type:fd,columns:[DNAME],dependentColumn:DEPTNO0}",
+ "{type:fd,columns:[DNAME],dependentColumn:DEPTNO}",
+ "{type:fd,columns:[DNAME],dependentColumn:LOC}",
+ "{type:fd,columns:[JOB],dependentColumn:COMM}",
+ "{type:fd,columns:[LOC],dependentColumn:DEPTNO0}",
+ "{type:fd,columns:[LOC],dependentColumn:DEPTNO}",
+ "{type:fd,columns:[LOC],dependentColumn:DNAME}",
+ "{type:fd,columns:[SAL],dependentColumn:DEPTNO0}",
+ "{type:fd,columns:[SAL],dependentColumn:DEPTNO}",
+ "{type:fd,columns:[SAL],dependentColumn:DNAME}",
+ "{type:fd,columns:[SAL],dependentColumn:JOB}",
+ "{type:fd,columns:[SAL],dependentColumn:LOC}",
+ "{type:fd,columns:[SAL],dependentColumn:MGR}",
+ "{type:rowCount,rowCount:14}",
+ "{type:unique,columns:[EMPNO]}",
+ "{type:unique,columns:[ENAME]}",
+ "{type:unique,columns:[HIREDATE,DEPTNO0]}",
+ "{type:unique,columns:[HIREDATE,DEPTNO]}",
+ "{type:unique,columns:[HIREDATE,DNAME]}",
+ "{type:unique,columns:[HIREDATE,LOC]}",
+ "{type:unique,columns:[HIREDATE,SAL]}",
+ "{type:unique,columns:[JOB,HIREDATE]}");
+ }
+
+ /** As {@link #testProfileScott()}, but prints only the most surprising
+ * distributions. */
+ @Test public void testProfileScott2() throws Exception {
+ scott().factory(Fluid.SIMPLE_FACTORY).unordered(
+ "{type:distribution,columns:[COMM],values:[0.00,300.00,500.00,1400.00],cardinality:5.0,nullCount:10,expectedCardinality:14.0,surprise:0.47368421052631576}",
+ "{type:distribution,columns:[DEPTNO,DEPTNO0],cardinality:3.0,expectedCardinality:7.269756624410332,surprise:0.41576025416819384}",
+ "{type:distribution,columns:[DEPTNO,DNAME],cardinality:3.0,expectedCardinality:7.269756624410332,surprise:0.41576025416819384}",
+ "{type:distribution,columns:[DEPTNO,LOC],cardinality:3.0,expectedCardinality:7.269756624410332,surprise:0.41576025416819384}",
+ "{type:distribution,columns:[DEPTNO0,DNAME],cardinality:3.0,expectedCardinality:7.269756624410332,surprise:0.41576025416819384}",
+ "{type:distribution,columns:[DEPTNO0,LOC],cardinality:3.0,expectedCardinality:7.269756624410332,surprise:0.41576025416819384}",
+ "{type:distribution,columns:[DEPTNO0],values:[10,20,30],cardinality:3.0,expectedCardinality:14.0,surprise:0.6470588235294118}",
+ "{type:distribution,columns:[DEPTNO],values:[10,20,30],cardinality:3.0,expectedCardinality:14.0,surprise:0.6470588235294118}",
+ "{type:distribution,columns:[DNAME,LOC],cardinality:3.0,expectedCardinality:7.269756624410332,surprise:0.41576025416819384}",
+ "{type:distribution,columns:[DNAME],values:[ACCOUNTING,RESEARCH,SALES],cardinality:3.0,expectedCardinality:14.0,surprise:0.6470588235294118}",
+ "{type:distribution,columns:[HIREDATE,COMM],cardinality:5.0,expectedCardinality:12.682618485430247,surprise:0.4344728973121492}",
+ "{type:distribution,columns:[HIREDATE],values:[1980-12-17,1981-01-05,1981-02-04,1981-02-20,1981-02-22,1981-06-09,1981-09-08,1981-09-28,1981-11-17,1981-12-03,1982-01-23,1987-04-19,1987-05-23],cardinality:13.0,expectedCardinality:14.0,surprise:0.037037037037037035}",
+ "{type:distribution,columns:[JOB],values:[ANALYST,CLERK,MANAGER,PRESIDENT,SALESMAN],cardinality:5.0,expectedCardinality:14.0,surprise:0.47368421052631576}",
+ "{type:distribution,columns:[LOC],values:[CHICAGO,DALLAS,NEWYORK],cardinality:3.0,expectedCardinality:14.0,surprise:0.6470588235294118}",
+ "{type:distribution,columns:[MGR,COMM],cardinality:5.0,expectedCardinality:11.675074674157162,surprise:0.400302535646339}",
+ "{type:distribution,columns:[MGR],values:[7566,7698,7782,7788,7839,7902],cardinality:7.0,nullCount:1,expectedCardinality:14.0,surprise:0.3333333333333333}",
+ "{type:distribution,columns:[SAL,COMM],cardinality:5.0,expectedCardinality:12.579960871109892,surprise:0.43117052004174}",
+ "{type:distribution,columns:[SAL],values:[800.00,950.00,1100.00,1250.00,1300.00,1500.00,1600.00,2450.00,2850.00,2975.00,3000.00,5000.00],cardinality:12.0,expectedCardinality:14.0,surprise:0.07692307692307693}",
+ "{type:distribution,columns:[],cardinality:1.0,expectedCardinality:1.0,surprise:0.0}");
+ }
+
+ /** As {@link #testProfileScott2()}, but uses the breadth-first profiler.
+ * Results should be the same, but are slightly different (extra EMPNO
+ * and ENAME distributions). */
+ @Test public void testProfileScott3() throws Exception {
+ scott().factory(Fluid.BETTER_FACTORY).unordered(
+ "{type:distribution,columns:[COMM],values:[0.00,300.00,500.00,1400.00],cardinality:5.0,nullCount:10,expectedCardinality:14.0,surprise:0.47368421052631576}",
+ "{type:distribution,columns:[DEPTNO,DEPTNO0,DNAME,LOC],cardinality:3.0,expectedCardinality:7.269756624410332,surprise:0.41576025416819384}",
+ "{type:distribution,columns:[DEPTNO,DEPTNO0],cardinality:3.0,expectedCardinality:7.269756624410332,surprise:0.41576025416819384}",
+ "{type:distribution,columns:[DEPTNO,DNAME],cardinality:3.0,expectedCardinality:7.269756624410332,surprise:0.41576025416819384}",
+ "{type:distribution,columns:[DEPTNO,LOC],cardinality:3.0,expectedCardinality:7.269756624410332,surprise:0.41576025416819384}",
+ "{type:distribution,columns:[DEPTNO0,DNAME,LOC],cardinality:3.0,expectedCardinality:14.0,surprise:0.6470588235294118}",
+ "{type:distribution,columns:[DEPTNO0],values:[10,20,30],cardinality:3.0,expectedCardinality:14.0,surprise:0.6470588235294118}",
+ "{type:distribution,columns:[DEPTNO],values:[10,20,30],cardinality:3.0,expectedCardinality:14.0,surprise:0.6470588235294118}",
+ "{type:distribution,columns:[DNAME],values:[ACCOUNTING,RESEARCH,SALES],cardinality:3.0,expectedCardinality:14.0,surprise:0.6470588235294118}",
+ "{type:distribution,columns:[EMPNO],values:[7369,7499,7521,7566,7654,7698,7782,7788,7839,7844,7876,7900,7902,7934],cardinality:14.0,expectedCardinality:14.0,surprise:0.0}",
+ "{type:distribution,columns:[ENAME],values:[ADAMS,ALLEN,BLAKE,CLARK,FORD,JAMES,JONES,KING,MARTIN,MILLER,SCOTT,SMITH,TURNER,WARD],cardinality:14.0,expectedCardinality:14.0,surprise:0.0}",
+ "{type:distribution,columns:[HIREDATE],values:[1980-12-17,1981-01-05,1981-02-04,1981-02-20,1981-02-22,1981-06-09,1981-09-08,1981-09-28,1981-11-17,1981-12-03,1982-01-23,1987-04-19,1987-05-23],cardinality:13.0,expectedCardinality:14.0,surprise:0.037037037037037035}",
+ "{type:distribution,columns:[JOB],values:[ANALYST,CLERK,MANAGER,PRESIDENT,SALESMAN],cardinality:5.0,expectedCardinality:14.0,surprise:0.47368421052631576}",
+ "{type:distribution,columns:[LOC],values:[CHICAGO,DALLAS,NEWYORK],cardinality:3.0,expectedCardinality:14.0,surprise:0.6470588235294118}",
+ "{type:distribution,columns:[MGR],values:[7566,7698,7782,7788,7839,7902],cardinality:7.0,nullCount:1,expectedCardinality:14.0,surprise:0.3333333333333333}",
+ "{type:distribution,columns:[SAL],values:[800.00,950.00,1100.00,1250.00,1300.00,1500.00,1600.00,2450.00,2850.00,2975.00,3000.00,5000.00],cardinality:12.0,expectedCardinality:14.0,surprise:0.07692307692307693}",
+ "{type:distribution,columns:[],cardinality:1.0,expectedCardinality:1.0,surprise:0.0}");
+ }
+
+ /** As {@link #testProfileScott3()}, but uses the breadth-first profiler
+ * and deems everything uninteresting. Only first-level combinations (those
+ * consisting of a single column) are computed. */
+ @Test public void testProfileScott4() throws Exception {
+ scott().factory(Fluid.INCURIOUS_PROFILER_FACTORY).unordered(
+ "{type:distribution,columns:[COMM],values:[0.00,300.00,500.00,1400.00],cardinality:5.0,nullCount:10,expectedCardinality:14.0,surprise:0.47368421052631576}",
+ "{type:distribution,columns:[DEPTNO0,DNAME,LOC],cardinality:3.0,expectedCardinality:14.0,surprise:0.6470588235294118}",
+ "{type:distribution,columns:[DEPTNO0],values:[10,20,30],cardinality:3.0,expectedCardinality:14.0,surprise:0.6470588235294118}",
+ "{type:distribution,columns:[DEPTNO],values:[10,20,30],cardinality:3.0,expectedCardinality:14.0,surprise:0.6470588235294118}",
+ "{type:distribution,columns:[DNAME],values:[ACCOUNTING,RESEARCH,SALES],cardinality:3.0,expectedCardinality:14.0,surprise:0.6470588235294118}",
+ "{type:distribution,columns:[EMPNO],values:[7369,7499,7521,7566,7654,7698,7782,7788,7839,7844,7876,7900,7902,7934],cardinality:14.0,expectedCardinality:14.0,surprise:0.0}",
+ "{type:distribution,columns:[ENAME],values:[ADAMS,ALLEN,BLAKE,CLARK,FORD,JAMES,JONES,KING,MARTIN,MILLER,SCOTT,SMITH,TURNER,WARD],cardinality:14.0,expectedCardinality:14.0,surprise:0.0}",
+ "{type:distribution,columns:[HIREDATE],values:[1980-12-17,1981-01-05,1981-02-04,1981-02-20,1981-02-22,1981-06-09,1981-09-08,1981-09-28,1981-11-17,1981-12-03,1982-01-23,1987-04-19,1987-05-23],cardinality:13.0,expectedCardinality:14.0,surprise:0.037037037037037035}",
+ "{type:distribution,columns:[JOB],values:[ANALYST,CLERK,MANAGER,PRESIDENT,SALESMAN],cardinality:5.0,expectedCardinality:14.0,surprise:0.47368421052631576}",
+ "{type:distribution,columns:[LOC],values:[CHICAGO,DALLAS,NEWYORK],cardinality:3.0,expectedCardinality:14.0,surprise:0.6470588235294118}",
+ "{type:distribution,columns:[MGR],values:[7566,7698,7782,7788,7839,7902],cardinality:7.0,nullCount:1,expectedCardinality:14.0,surprise:0.3333333333333333}",
+ "{type:distribution,columns:[SAL],values:[800.00,950.00,1100.00,1250.00,1300.00,1500.00,1600.00,2450.00,2850.00,2975.00,3000.00,5000.00],cardinality:12.0,expectedCardinality:14.0,surprise:0.07692307692307693}",
+ "{type:distribution,columns:[],cardinality:1.0,expectedCardinality:1.0,surprise:0.0}");
+ }
+
+ /** As {@link #testProfileScott3()}, but uses the breadth-first profiler. */
+ @Ignore
+ @Test public void testProfileScott5() throws Exception {
+ scott().factory(Fluid.PROFILER_FACTORY).unordered(
+ "{type:distribution,columns:[COMM],values:[0.00,300.00,500.00,1400.00],cardinality:5.0,nullCount:10,expectedCardinality:14.0,surprise:0.47368421052631576}",
+ "{type:distribution,columns:[DEPTNO,DEPTNO0,DNAME,LOC],cardinality:3.0,expectedCardinality:7.269756624410332,surprise:0.41576025416819384}",
+ "{type:distribution,columns:[DEPTNO,DEPTNO0],cardinality:3.0,expectedCardinality:7.269756624410332,surprise:0.41576025416819384}",
+ "{type:distribution,columns:[DEPTNO,DNAME],cardinality:3.0,expectedCardinality:7.269756624410332,surprise:0.41576025416819384}",
+ "{type:distribution,columns:[DEPTNO,LOC],cardinality:3.0,expectedCardinality:7.269756624410332,surprise:0.41576025416819384}",
+ "{type:distribution,columns:[DEPTNO0,DNAME,LOC],cardinality:3.0,expectedCardinality:14.0,surprise:0.6470588235294118}",
+ "{type:distribution,columns:[DEPTNO0],values:[10,20,30],cardinality:3.0,expectedCardinality:14.0,surprise:0.6470588235294118}",
+ "{type:distribution,columns:[DEPTNO],values:[10,20,30],cardinality:3.0,expectedCardinality:14.0,surprise:0.6470588235294118}",
+ "{type:distribution,columns:[DNAME],values:[ACCOUNTING,RESEARCH,SALES],cardinality:3.0,expectedCardinality:14.0,surprise:0.6470588235294118}",
+ "{type:distribution,columns:[EMPNO],values:[7369,7499,7521,7566,7654,7698,7782,7788,7839,7844,7876,7900,7902,7934],cardinality:14.0,expectedCardinality:14.0,surprise:0.0}",
+ "{type:distribution,columns:[ENAME],values:[ADAMS,ALLEN,BLAKE,CLARK,FORD,JAMES,JONES,KING,MARTIN,MILLER,SCOTT,SMITH,TURNER,WARD],cardinality:14.0,expectedCardinality:14.0,surprise:0.0}",
+ "{type:distribution,columns:[HIREDATE],values:[1980-12-17,1981-01-05,1981-02-04,1981-02-20,1981-02-22,1981-06-09,1981-09-08,1981-09-28,1981-11-17,1981-12-03,1982-01-23,1987-04-19,1987-05-23],cardinality:13.0,expectedCardinality:14.0,surprise:0.037037037037037035}",
+ "{type:distribution,columns:[JOB],values:[ANALYST,CLERK,MANAGER,PRESIDENT,SALESMAN],cardinality:5.0,expectedCardinality:14.0,surprise:0.47368421052631576}",
+ "{type:distribution,columns:[LOC],values:[CHICAGO,DALLAS,NEWYORK],cardinality:3.0,expectedCardinality:14.0,surprise:0.6470588235294118}",
+ "{type:distribution,columns:[MGR],values:[7566,7698,7782,7788,7839,7902],cardinality:7.0,nullCount:1,expectedCardinality:14.0,surprise:0.3333333333333333}",
+ "{type:distribution,columns:[SAL],values:[800.00,950.00,1100.00,1250.00,1300.00,1500.00,1600.00,2450.00,2850.00,2975.00,3000.00,5000.00],cardinality:12.0,expectedCardinality:14.0,surprise:0.07692307692307693}",
+ "{type:distribution,columns:[],cardinality:1.0,expectedCardinality:1.0,surprise:0.0}");
+ }
+
+ /** Profiles a star-join query on the Foodmart schema using the breadth-first
+ * profiler. */
+ @Ignore
+ @Test public void testProfileFoodmart() throws Exception {
+ foodmart().factory(Fluid.PROFILER_FACTORY).unordered(
+ "{type:distribution,columns:[brand_name],cardinality:111.0,expectedCardinality:86837.0,surprise:0.9974467497814786}",
+ "{type:distribution,columns:[cases_per_pallet],values:[5,6,7,8,9,10,11,12,13,14],cardinality:10.0,expectedCardinality:86837.0,surprise:0.9997697099496816}",
+ "{type:distribution,columns:[day_of_month],cardinality:30.0,expectedCardinality:86837.0,surprise:0.9993092889129359}",
+ "{type:distribution,columns:[fiscal_period],values:[],cardinality:1.0,nullCount:86837,expectedCardinality:86837.0,surprise:0.999976968608213}",
+ "{type:distribution,columns:[low_fat],values:[false,true],cardinality:2.0,expectedCardinality:86837.0,surprise:0.9999539377468649}",
+ "{type:distribution,columns:[month_of_year],values:[1,2,3,4,5,6,7,8,9,10,11,12],cardinality:12.0,expectedCardinality:86837.0,surprise:0.9997236583034923}",
+ "{type:distribution,columns:[product_category],cardinality:45.0,expectedCardinality:86837.0,surprise:0.9989641122441932}",
+ "{type:distribution,columns:[product_class_id0,product_subcategory,product_category,product_department,product_family],cardinality:102.0,expectedCardinality:86837.0,surprise:0.997653527185728}",
+ "{type:distribution,columns:[product_class_id0],cardinality:102.0,expectedCardinality:86837.0,surprise:0.997653527185728}",
+ "{type:distribution,columns:[product_class_id],cardinality:102.0,expectedCardinality:86837.0,surprise:0.997653527185728}",
+ "{type:distribution,columns:[product_department],cardinality:22.0,expectedCardinality:86837.0,surprise:0.9994934318838578}",
+ "{type:distribution,columns:[product_family],values:[Drink,Food,Non-Consumable],cardinality:3.0,expectedCardinality:86837.0,surprise:0.9999309074159374}",
+ "{type:distribution,columns:[product_subcategory],cardinality:102.0,expectedCardinality:86837.0,surprise:0.997653527185728}",
+ "{type:distribution,columns:[quarter],values:[Q1,Q2,Q3,Q4],cardinality:4.0,expectedCardinality:86837.0,surprise:0.9999078776154121}",
+ "{type:distribution,columns:[recyclable_package],values:[false,true],cardinality:2.0,expectedCardinality:86837.0,surprise:0.9999539377468649}",
+ "{type:distribution,columns:[store_cost,fiscal_period],cardinality:10601.0,nullCount:86724,expectedCardinality:10.0,surprise:0.9981151635095655}",
+ "{type:distribution,columns:[store_cost,low_fat],cardinality:17673.0,expectedCardinality:20.0,surprise:0.99773921890013}",
+ "{type:distribution,columns:[store_cost,product_family],cardinality:19453.0,expectedCardinality:30.0,surprise:0.9969203921367346}",
+ "{type:distribution,columns:[store_cost,quarter],cardinality:29590.0,expectedCardinality:40.0,surprise:0.9973000337495781}",
+ "{type:distribution,columns:[store_cost,recyclable_package],cardinality:17847.0,expectedCardinality:20.0,surprise:0.9977612357978396}",
+ "{type:distribution,columns:[store_cost,the_year],cardinality:10944.0,expectedCardinality:10.0,surprise:0.9981741829468688}",
+ "{type:distribution,columns:[store_cost],cardinality:10.0,expectedCardinality:86837.0,surprise:0.9997697099496816}",
+ "{type:distribution,columns:[store_id],values:[2,3,6,7,11,13,14,15,16,17,22,23,24],cardinality:13.0,expectedCardinality:86837.0,surprise:0.9997006332757629}",
+ "{type:distribution,columns:[store_sales],cardinality:21.0,expectedCardinality:86837.0,surprise:0.999516452140275}",
+ "{type:distribution,columns:[the_day],values:[Friday,Monday,Saturday,Sunday,Thursday,Tuesday,Wednesday],cardinality:7.0,expectedCardinality:86837.0,surprise:0.9998387913960665}",
+ "{type:distribution,columns:[the_month],values:[April,August,December,February,January,July,June,March,May,November,October,September],cardinality:12.0,expectedCardinality:86837.0,surprise:0.9997236583034923}",
+ "{type:distribution,columns:[the_year],values:[1997],cardinality:1.0,expectedCardinality:86837.0,surprise:0.999976968608213}",
+ "{type:distribution,columns:[unit_sales],values:[1.0000,2.0000,3.0000,4.0000,5.0000,6.0000],cardinality:6.0,expectedCardinality:86837.0,surprise:0.999861819605495}",
+ "{type:distribution,columns:[units_per_case],cardinality:36.0,expectedCardinality:86837.0,surprise:0.9991712039413857}",
+ "{type:distribution,columns:[week_of_year],cardinality:52.0,expectedCardinality:86837.0,surprise:0.9988030705843087}",
+ "{type:distribution,columns:[],cardinality:1.0,expectedCardinality:1.0,surprise:0.0}");
+ }
+
+ /** Tests
+ * {@link org.apache.calcite.profile.ProfilerImpl.SurpriseQueue}. */
+ @Test public void testSurpriseQueue() {
+ ProfilerImpl.SurpriseQueue q = new ProfilerImpl.SurpriseQueue(4, 3);
+ assertThat(q.offer(2), is(true));
+ assertThat(q.toString(), is("min: 2.0, contents: [2.0]"));
+ assertThat(q.isValid(), is(true));
+
+ assertThat(q.offer(4), is(true));
+ assertThat(q.toString(), is("min: 2.0, contents: [2.0, 4.0]"));
+ assertThat(q.isValid(), is(true));
+
+ // Since we're in the warm-up period, a value lower than the minimum is
+ // accepted.
+ assertThat(q.offer(1), is(true));
+ assertThat(q.toString(), is("min: 1.0, contents: [2.0, 4.0, 1.0]"));
+ assertThat(q.isValid(), is(true));
+
+ assertThat(q.offer(5), is(true));
+ assertThat(q.toString(), is("min: 1.0, contents: [4.0, 1.0, 5.0]"));
+ assertThat(q.isValid(), is(true));
+
+ assertThat(q.offer(3), is(true));
+ assertThat(q.toString(), is("min: 1.0, contents: [1.0, 5.0, 3.0]"));
+ assertThat(q.isValid(), is(true));
+
+ // Duplicate entry
+ assertThat(q.offer(5), is(true));
+ assertThat(q.toString(), is("min: 3.0, contents: [5.0, 3.0, 5.0]"));
+ assertThat(q.isValid(), is(true));
+
+ // Now that the list is full, a value below the minimum is refused.
+ // "offer" returns false, and the value is not added to the queue.
+ // Thus the median never decreases.
+ assertThat(q.offer(2), is(false));
+ assertThat(q.toString(), is("min: 3.0, contents: [5.0, 3.0, 5.0]"));
+ assertThat(q.isValid(), is(true));
+
+ // Same applies for a value equal to the minimum.
+ assertThat(q.offer(3), is(false));
+ assertThat(q.toString(), is("min: 3.0, contents: [5.0, 3.0, 5.0]"));
+ assertThat(q.isValid(), is(true));
+
+ // Add a value that is above the minimum.
+ assertThat(q.offer(4.5), is(true));
+ assertThat(q.toString(), is("min: 3.0, contents: [3.0, 5.0, 4.5]"));
+ assertThat(q.isValid(), is(true));
+ }
+
+ private Fluid scott() throws Exception {
+ final String sql = "select * from \"scott\".emp\n"
+ + "join \"scott\".dept using (deptno)";
+ return sql(sql)
+ .where(Fluid.STATISTIC_PREDICATE)
+ .sort(Fluid.ORDERING.reverse())
+ .limit(30)
+ .project(Fluid.EXTENDED_COLUMNS);
+ }
+
+ private Fluid foodmart() throws Exception {
+ final String sql = "select \"s\".*, \"p\".*, \"t\".*, \"pc\".*\n"
+ + "from \"foodmart\".\"sales_fact_1997\" as \"s\"\n"
+ + "join \"foodmart\".\"product\" as \"p\" using (\"product_id\")\n"
+ + "join \"foodmart\".\"time_by_day\" as \"t\" using (\"time_id\")\n"
+ + "join \"foodmart\".\"product_class\" as \"pc\"\n"
+ + " on \"p\".\"product_class_id\" = \"pc\".\"product_class_id\"\n";
+ return sql(sql)
+ .config(CalciteAssert.Config.JDBC_FOODMART)
+ .where(Fluid.STATISTIC_PREDICATE)
+ .sort(Fluid.ORDERING.reverse())
+ .limit(30)
+ .project(Fluid.EXTENDED_COLUMNS);
+ }
+
+ private static Fluid sql(String sql) {
+ return new Fluid(CalciteAssert.Config.SCOTT, sql, Fluid.SIMPLE_FACTORY,
+ Predicates.<Profiler.Statistic>alwaysTrue(), null, -1,
+ Fluid.DEFAULT_COLUMNS);
+ }
+
+ /** Fluid interface for writing profiler test cases. */
+ private static class Fluid {
+ static final Supplier<Profiler> SIMPLE_FACTORY =
+ new Supplier<Profiler>() {
+ public Profiler get() {
+ return new SimpleProfiler();
+ }
+ };
+
+ static final Supplier<Profiler> BETTER_FACTORY =
+ new Supplier<Profiler>() {
+ public Profiler get() {
+ final Predicate<Pair<ProfilerImpl.Space, Profiler.Column>>
+ predicate = Predicates.alwaysTrue();
+ return new ProfilerImpl(600, 200, predicate);
+ }
+ };
+
+ static final Ordering<Profiler.Statistic> ORDERING =
+ new Ordering<Profiler.Statistic>() {
+ public int compare(Profiler.Statistic left,
+ Profiler.Statistic right) {
+ int c = left.getClass().getSimpleName()
+ .compareTo(right.getClass().getSimpleName());
+ if (c == 0
+ && left instanceof Profiler.Distribution
+ && right instanceof Profiler.Distribution) {
+ final Profiler.Distribution d0 = (Profiler.Distribution) left;
+ final Profiler.Distribution d1 = (Profiler.Distribution) right;
+ c = Double.compare(d0.surprise(), d1.surprise());
+ if (c == 0) {
+ c = d0.columns.toString().compareTo(d1.columns.toString());
+ }
+ }
+ return c;
+ }
+ };
+
+ static final Predicate<Profiler.Statistic> STATISTIC_PREDICATE =
+ new PredicateImpl<Profiler.Statistic>() {
+ public boolean test(Profiler.Statistic statistic) {
+ // Include distributions of zero columns (the grand total)
+ // and singleton columns, plus "surprising" distributions
+ // (with significantly higher NDVs than predicted from their
+ // constituent columns).
+ return statistic instanceof Profiler.Distribution
+ && (((Profiler.Distribution) statistic).columns.size() < 2
+ || ((Profiler.Distribution) statistic).surprise() > 0.4D)
+ && ((Profiler.Distribution) statistic).minimal;
+ }
+ };
+
+ static final List<String> DEFAULT_COLUMNS =
+ ImmutableList.of("type", "distribution", "columns", "cardinality",
+ "values", "nullCount", "dependentColumn", "rowCount");
+
+ static final List<String> EXTENDED_COLUMNS =
+ ImmutableList.<String>builder().addAll(DEFAULT_COLUMNS)
+ .add("expectedCardinality", "surprise")
+ .build();
+
+ private static final Supplier<Profiler> PROFILER_FACTORY =
+ new Supplier<Profiler>() {
+ public Profiler get() {
+ return new ProfilerImpl(7500, 100,
+ new PredicateImpl<Pair<ProfilerImpl.Space, Profiler.Column>>() {
+ public boolean test(
+ Pair<ProfilerImpl.Space, Profiler.Column> p) {
+ final Profiler.Distribution distribution =
+ p.left.distribution();
+ if (distribution == null) {
+ // We don't have a distribution yet, because this space
+ // has not yet been evaluated. Let's do it anyway.
+ return true;
+ }
+ return distribution.surprise() >= 0.3D;
+ }
+ });
+ }
+ };
+
+ private static final Supplier<Profiler> INCURIOUS_PROFILER_FACTORY =
+ new Supplier<Profiler>() {
+ public Profiler get() {
+ final Predicate<Pair<ProfilerImpl.Space, Profiler.Column>> p =
+ Predicates.alwaysFalse();
+ return new ProfilerImpl(10, 200, p);
+ }
+ };
+
+ private final String sql;
+ private final List<String> columns;
+ private final Comparator<Profiler.Statistic> comparator;
+ private final int limit;
+ private final Predicate<Profiler.Statistic> predicate;
+ private final Supplier<Profiler> factory;
+ private final CalciteAssert.Config config;
+
+ Fluid(CalciteAssert.Config config, String sql, Supplier<Profiler> factory,
+ Predicate<Profiler.Statistic> predicate,
+ Comparator<Profiler.Statistic> comparator, int limit,
+ List<String> columns) {
+ this.sql = Preconditions.checkNotNull(sql);
+ this.factory = Preconditions.checkNotNull(factory);
+ this.columns = ImmutableList.copyOf(columns);
+ this.predicate = Preconditions.checkNotNull(predicate);
+ this.comparator = comparator; // null means sort on JSON representation
+ this.limit = limit;
+ this.config = config;
+ }
+
+ Fluid config(CalciteAssert.Config config) {
+ return new Fluid(config, sql, factory, predicate, comparator, limit,
+ columns);
+ }
+
+ Fluid factory(Supplier<Profiler> factory) {
+ return new Fluid(config, sql, factory, predicate, comparator, limit,
+ columns);
+ }
+
+ Fluid project(List<String> columns) {
+ return new Fluid(config, sql, factory, predicate, comparator, limit,
+ columns);
+ }
+
+ Fluid sort(Ordering<Profiler.Statistic> comparator) {
+ return new Fluid(config, sql, factory, predicate, comparator, limit,
+ columns);
+ }
+
+ Fluid limit(int limit) {
+ return new Fluid(config, sql, factory, predicate, comparator, limit,
+ columns);
+ }
+
+ Fluid where(Predicate<Profiler.Statistic> predicate) {
+ return new Fluid(config, sql, factory, predicate, comparator, limit,
+ columns);
+ }
+
+ Fluid unordered(String... lines) throws Exception {
+ return check(Matchers.equalsUnordered(lines));
+ }
+
+ public Fluid check(final Matcher<Iterable<String>> matcher)
+ throws Exception {
+ CalciteAssert.that(config)
+ .doWithConnection(new Function<CalciteConnection, Void>() {
+ public Void apply(CalciteConnection c) {
+ try (PreparedStatement s = c.prepareStatement(sql)) {
+ final ResultSetMetaData m = s.getMetaData();
+ final List<Profiler.Column> columns = new ArrayList<>();
+ final int columnCount = m.getColumnCount();
+ for (int i = 0; i < columnCount; i++) {
+ columns.add(new Profiler.Column(i, m.getColumnLabel(i + 1)));
+ }
+
+ // Create an initial group for each table in the query.
+ // Columns in the same table will tend to have the same
+ // cardinality as the table, and as the table's primary key.
+ final Multimap<String, Integer> groups = HashMultimap.create();
+ for (int i = 0; i < m.getColumnCount(); i++) {
+ groups.put(m.getTableName(i + 1), i);
+ }
+ final SortedSet<ImmutableBitSet> initialGroups =
+ new TreeSet<>();
+ for (Collection<Integer> integers : groups.asMap().values()) {
+ initialGroups.add(ImmutableBitSet.of(integers));
+ }
+ final Profiler p = factory.get();
+ final Enumerable<List<Comparable>> rows = getRows(s);
+ final Profiler.Profile profile =
+ p.profile(rows, columns, initialGroups);
+ final List<Profiler.Statistic> statistics =
+ ImmutableList.copyOf(
+ Iterables.filter(profile.statistics(), predicate));
+
+ // If no comparator specified, use the function that converts to
+ // JSON strings
+ final Function<Profiler.Statistic, String> toJson =
+ toJsonFunction();
+ Ordering<Profiler.Statistic> comp = comparator != null
+ ? Ordering.from(comparator)
+ : Ordering.natural().onResultOf(toJson);
+ ImmutableList<Profiler.Statistic> statistics2 =
+ comp.immutableSortedCopy(statistics);
+ if (limit >= 0 && limit < statistics2.size()) {
+ statistics2 = statistics2.subList(0, limit);
+ }
+
+ final List<String> strings =
+ Lists.transform(statistics2, toJson);
+ assertThat(strings, matcher);
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ return null;
+ }
+ });
+ return this;
+ }
+
+ /** Returns a function that converts a statistic to a JSON string. */
+ Function<Profiler.Statistic, String> toJsonFunction() {
+ return new Function<Profiler.Statistic, String>() {
+ final JsonBuilder jb = new JsonBuilder();
+
+ public String apply(Profiler.Statistic statistic) {
+ Object map = statistic.toMap(jb);
+ if (map instanceof Map) {
+ @SuppressWarnings("unchecked")
+ final Map<String, Object> map1 = (Map) map;
+ map1.keySet().retainAll(Fluid.this.columns);
+ }
+ final String json = jb.toJsonString(map);
+ return json.replaceAll("\n", "").replaceAll(" ", "")
+ .replaceAll("\"", "");
+ }
+ };
+ }
+
+ private Enumerable<List<Comparable>> getRows(final PreparedStatement s) {
+ return new AbstractEnumerable<List<Comparable>>() {
+ public Enumerator<List<Comparable>> enumerator() {
+ try {
+ final ResultSet r = s.executeQuery();
+ return getListEnumerator(r, r.getMetaData().getColumnCount());
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ };
+ }
+
+ private Enumerator<List<Comparable>> getListEnumerator(
+ final ResultSet r, final int columnCount) {
+ return new Enumerator<List<Comparable>>() {
+ final Comparable[] values = new Comparable[columnCount];
+
+ public List<Comparable> current() {
+ for (int i = 0; i < columnCount; i++) {
+ try {
+ final Comparable value = (Comparable) r.getObject(i + 1);
+ values[i] = NullSentinel.mask(value);
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ return ImmutableList.copyOf(values);
+ }
+
+ public boolean moveNext() {
+ try {
+ return r.next();
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public void reset() {
+ }
+
+ public void close() {
+ try {
+ r.close();
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ };
+ }
+ }
+}
+
+// End ProfilerTest.java
http://git-wip-us.apache.org/repos/asf/calcite/blob/dad58186/core/src/test/java/org/apache/calcite/test/CalciteSuite.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/calcite/test/CalciteSuite.java b/core/src/test/java/org/apache/calcite/test/CalciteSuite.java
index d577057..585e8ed 100644
--- a/core/src/test/java/org/apache/calcite/test/CalciteSuite.java
+++ b/core/src/test/java/org/apache/calcite/test/CalciteSuite.java
@@ -28,6 +28,7 @@ import org.apache.calcite.plan.volcano.TraitPropagationTest;
import org.apache.calcite.plan.volcano.VolcanoPlannerTest;
import org.apache.calcite.plan.volcano.VolcanoPlannerTraitTest;
import org.apache.calcite.prepare.LookupOperatorOverloadsTest;
+import org.apache.calcite.profile.ProfilerTest;
import org.apache.calcite.rel.RelCollationTest;
import org.apache.calcite.rel.RelDistributionTest;
import org.apache.calcite.rel.rel2sql.RelToSqlConverterTest;
@@ -156,6 +157,7 @@ import org.junit.runners.Suite;
LinqFrontJdbcBackTest.class,
JdbcFrontJdbcBackLinqMiddleTest.class,
CalciteSqlOperatorTest.class,
+ ProfilerTest.class,
LatticeTest.class,
ReflectiveSchemaTest.class,
JdbcTest.class,
http://git-wip-us.apache.org/repos/asf/calcite/blob/dad58186/core/src/test/java/org/apache/calcite/test/FoodMartLatticeStatisticProvider.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/calcite/test/FoodMartLatticeStatisticProvider.java b/core/src/test/java/org/apache/calcite/test/FoodMartLatticeStatisticProvider.java
index 3f36bf9..4eae872 100644
--- a/core/src/test/java/org/apache/calcite/test/FoodMartLatticeStatisticProvider.java
+++ b/core/src/test/java/org/apache/calcite/test/FoodMartLatticeStatisticProvider.java
@@ -23,6 +23,8 @@ import org.apache.calcite.materialize.Lattices;
import com.google.common.collect.ImmutableMap;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Map;
/**
@@ -33,10 +35,15 @@ import java.util.Map;
*/
public class FoodMartLatticeStatisticProvider
extends DelegatingLatticeStatisticProvider {
- public static final FoodMartLatticeStatisticProvider INSTANCE =
- new FoodMartLatticeStatisticProvider(Lattices.CACHED_SQL);
+ public static final FoodMartLatticeStatisticProvider.Factory FACTORY =
+ new Factory() {
+ public LatticeStatisticProvider apply(Lattice lattice) {
+ return new FoodMartLatticeStatisticProvider(lattice,
+ Lattices.CACHED_SQL.apply(lattice));
+ }
+ };
- public static final Map<String, Integer> CARDINALITY_MAP =
+ private static final Map<String, Integer> CARDINALITY_MAP =
ImmutableMap.<String, Integer>builder()
.put("brand_name", 111)
.put("cases_per_pallet", 10)
@@ -75,18 +82,29 @@ public class FoodMartLatticeStatisticProvider
.put("week_of_year", 52)
.build();
- private FoodMartLatticeStatisticProvider(LatticeStatisticProvider provider) {
+ private final Lattice lattice;
+
+ private FoodMartLatticeStatisticProvider(Lattice lattice,
+ LatticeStatisticProvider provider) {
super(provider);
+ this.lattice = lattice;
}
- /** Returns an estimate of the number of distinct values in a column. */
- public int cardinality(Lattice lattice, Lattice.Column column) {
+ private int cardinality(Lattice.Column column) {
final Integer integer = CARDINALITY_MAP.get(column.alias);
if (integer != null && integer > 0) {
return integer;
}
return column.alias.length();
}
+
+ @Override public double cardinality(List<Lattice.Column> columns) {
+ final List<Double> cardinalityList = new ArrayList<>();
+ for (Lattice.Column column : columns) {
+ cardinalityList.add((double) cardinality(column));
+ }
+ return Lattice.getRowCount(lattice.getFactRowCount(), cardinalityList);
+ }
}
// End FoodMartLatticeStatisticProvider.java
http://git-wip-us.apache.org/repos/asf/calcite/blob/dad58186/core/src/test/java/org/apache/calcite/test/LatticeTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/calcite/test/LatticeTest.java b/core/src/test/java/org/apache/calcite/test/LatticeTest.java
index 54f4ea7..473e5f5 100644
--- a/core/src/test/java/org/apache/calcite/test/LatticeTest.java
+++ b/core/src/test/java/org/apache/calcite/test/LatticeTest.java
@@ -16,11 +16,16 @@
*/
package org.apache.calcite.test;
+import org.apache.calcite.jdbc.CalciteConnection;
+import org.apache.calcite.jdbc.CalciteSchema;
+import org.apache.calcite.materialize.Lattice;
import org.apache.calcite.materialize.Lattices;
import org.apache.calcite.materialize.MaterializationService;
import org.apache.calcite.plan.RelOptUtil;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.runtime.Hook;
+import org.apache.calcite.schema.SchemaPlus;
+import org.apache.calcite.util.ImmutableBitSet;
import org.apache.calcite.util.TestUtil;
import org.apache.calcite.util.Util;
@@ -29,6 +34,7 @@ import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
+import org.junit.Assume;
import org.junit.Ignore;
import org.junit.Test;
@@ -39,11 +45,15 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.Arrays;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
+import static org.apache.calcite.test.Matchers.within;
+
import static org.hamcrest.CoreMatchers.anyOf;
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.core.Is.is;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertThat;
@@ -110,8 +120,8 @@ public class LatticeTest {
+ " } ]\n"
+ "}\n";
- private CalciteAssert.AssertThat modelWithLattice(String name, String sql,
- String... extras) {
+ private static CalciteAssert.AssertThat modelWithLattice(String name,
+ String sql, String... extras) {
final StringBuilder buf = new StringBuilder("{ name: '")
.append(name)
.append("', sql: ")
@@ -123,7 +133,8 @@ public class LatticeTest {
return modelWithLattices(buf.toString());
}
- private CalciteAssert.AssertThat modelWithLattices(String... lattices) {
+ private static CalciteAssert.AssertThat modelWithLattices(
+ String... lattices) {
final Class<JdbcTest.EmpDeptTableFactory> clazz =
JdbcTest.EmpDeptTableFactory.class;
return CalciteAssert.model(""
@@ -153,6 +164,36 @@ public class LatticeTest {
/** Tests that it's OK for a lattice to have the same name as a table in the
* schema. */
+ @Test public void testLatticeSql() throws Exception {
+ modelWithLattice("EMPLOYEES", "select * from \"foodmart\".\"days\"")
+ .doWithConnection(new Function<CalciteConnection, Void>() {
+ public Void apply(CalciteConnection c) {
+ final SchemaPlus schema = c.getRootSchema();
+ final SchemaPlus adhoc = schema.getSubSchema("adhoc");
+ assertThat(adhoc.getTableNames().contains("EMPLOYEES"), is(true));
+ final Map.Entry<String, CalciteSchema.LatticeEntry> entry =
+ adhoc.unwrap(CalciteSchema.class).getLatticeMap().firstEntry();
+ final Lattice lattice = entry.getValue().getLattice();
+ final String sql = "SELECT \"days\".\"day\"\n"
+ + "FROM \"foodmart\".\"days\" AS \"days\"\n"
+ + "GROUP BY \"days\".\"day\"";
+ assertThat(
+ lattice.sql(ImmutableBitSet.of(0),
+ ImmutableList.<Lattice.Measure>of()), is(sql));
+ final String sql2 = "SELECT"
+ + " \"days\".\"day\", \"days\".\"week_day\"\n"
+ + "FROM \"foodmart\".\"days\" AS \"days\"";
+ assertThat(
+ lattice.sql(ImmutableBitSet.of(0, 1), false,
+ ImmutableList.<Lattice.Measure>of()),
+ is(sql2));
+ return null;
+ }
+ });
+ }
+
+ /** Tests that it's OK for a lattice to have the same name as a table in the
+ * schema. */
@Test public void testLatticeWithSameNameAsTable() {
modelWithLattice("EMPLOYEES", "select * from \"foodmart\".\"days\"")
.query("select count(*) from EMPLOYEES")
@@ -377,28 +418,57 @@ public class LatticeTest {
* Use optimization algorithm to suggest which tiles of a lattice to
* materialize</a>. */
@Test public void testTileAlgorithm() {
- checkTileAlgorithm(FoodMartLatticeStatisticProvider.class.getCanonicalName(),
- "EnumerableAggregate(group=[{2, 3}])\n"
- + " EnumerableTableScan(table=[[adhoc, m{16, 17, 27, 31}]])");
+ final String explain = "EnumerableAggregate(group=[{2, 3}])\n"
+ + " EnumerableTableScan(table=[[adhoc, m{16, 17, 27, 31, 32, 37}]])";
+ checkTileAlgorithm(
+ FoodMartLatticeStatisticProvider.class.getCanonicalName() + "#FACTORY",
+ explain);
}
+ /** As {@link #testTileAlgorithm()}, but uses the
+ * {@link Lattices#CACHED_SQL} statistics provider. */
@Test public void testTileAlgorithm2() {
// Different explain than above, but note that it still selects columns
// (27, 31).
+ final String explain = "EnumerableAggregate(group=[{0, 1}])\n"
+ + " EnumerableTableScan(table=[[adhoc, m{27, 31, 32, 36, 37}]";
checkTileAlgorithm(Lattices.class.getCanonicalName() + "#CACHED_SQL",
- "EnumerableAggregate(group=[{0, 1}])\n"
- + " EnumerableTableScan(table=[[adhoc, m{27, 31, 32, 36, 37}]");
+ explain);
+ }
+
+ /** As {@link #testTileAlgorithm()}, but uses the
+ * {@link Lattices#PROFILER} statistics provider. */
+ @Test public void testTileAlgorithm3() {
+ Assume.assumeTrue("Yahoo sketches requires JDK 8 or higher",
+ TestUtil.getJavaMajorVersion() >= 8);
+ final String explain = "EnumerableAggregate(group=[{0, 1}])\n"
+ + " EnumerableTableScan(table=[[adhoc, m{27, 31, 32, 36, 37}]";
+ checkTileAlgorithm(Lattices.class.getCanonicalName() + "#PROFILER",
+ explain);
}
private void checkTileAlgorithm(String statisticProvider,
String expectedExplain) {
MaterializationService.setThreadLocal();
MaterializationService.instance().clear();
- foodmartModel(
- " auto: false,\n"
+ foodmartLatticeModel(statisticProvider)
+ .query("select distinct t.\"the_year\", t.\"quarter\"\n"
+ + "from \"foodmart\".\"sales_fact_1997\" as s\n"
+ + "join \"foodmart\".\"time_by_day\" as t using (\"time_id\")\n")
+ .enableMaterializations(true)
+ .explainContains(expectedExplain)
+ .returnsUnordered("the_year=1997; quarter=Q1",
+ "the_year=1997; quarter=Q2",
+ "the_year=1997; quarter=Q3",
+ "the_year=1997; quarter=Q4");
+ }
+
+ private static CalciteAssert.AssertThat foodmartLatticeModel(
+ String statisticProvider) {
+ return foodmartModel(" auto: false,\n"
+ " algorithm: true,\n"
+ " algorithmMaxMillis: -1,\n"
- + " rowCountEstimate: 86000,\n"
+ + " rowCountEstimate: 87000,\n"
+ " defaultMeasures: [ {\n"
+ " agg: 'sum',\n"
+ " args: 'unit_sales'\n"
@@ -414,17 +484,7 @@ public class LatticeTest {
+ " tiles: [ {\n"
+ " dimensions: [ 'the_year', ['t', 'quarter'] ],\n"
+ " measures: [ ]\n"
- + " } ]\n")
- .query("select distinct t.\"the_year\", t.\"quarter\"\n"
- + "from \"foodmart\".\"sales_fact_1997\" as s\n"
- + "join \"foodmart\".\"time_by_day\" as t using (\"time_id\")\n")
- .enableMaterializations(true)
- .explainContains(expectedExplain)
- .returnsUnordered("the_year=1997; quarter=Q1",
- "the_year=1997; quarter=Q2",
- "the_year=1997; quarter=Q3",
- "the_year=1997; quarter=Q4")
- .returnsCount(4);
+ + " } ]\n");
}
/** Tests a query that is created within {@link #testTileAlgorithm()}. */
@@ -704,7 +764,7 @@ public class LatticeTest {
.returns("EXPR$0=1\n");
}
- private CalciteAssert.AssertThat foodmartModel(String... extras) {
+ private static CalciteAssert.AssertThat foodmartModel(String... extras) {
return modelWithLattice("star",
"select 1 from \"foodmart\".\"sales_fact_1997\" as \"s\"\n"
+ "join \"foodmart\".\"product\" as \"p\" using (\"product_id\")\n"
@@ -741,6 +801,17 @@ public class LatticeTest {
System.out.println(CalciteAssert.toString(resultSet));
connection.close();
}
+
+ /** Unit test for {@link Lattice#getRowCount(double, List)}. */
+ @Test public void testColumnCount() {
+ assertThat(Lattice.getRowCount(10, 2, 3), within(5.03D, 0.01D));
+ assertThat(Lattice.getRowCount(10, 9, 8), within(9.4D, 0.01D));
+ assertThat(Lattice.getRowCount(100, 9, 8), within(54.2D, 0.1D));
+ assertThat(Lattice.getRowCount(1000, 9, 8), within(72D, 0.01D));
+ assertThat(Lattice.getRowCount(1000, 1, 1), is(1D));
+ assertThat(Lattice.getRowCount(1, 3, 5), within(1D, 0.01D));
+ assertThat(Lattice.getRowCount(1, 3, 5, 13, 4831), within(1D, 0.01D));
+ }
}
// End LatticeTest.java
http://git-wip-us.apache.org/repos/asf/calcite/blob/dad58186/core/src/test/java/org/apache/calcite/test/Matchers.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/calcite/test/Matchers.java b/core/src/test/java/org/apache/calcite/test/Matchers.java
index fc9e0fd..08733a2 100644
--- a/core/src/test/java/org/apache/calcite/test/Matchers.java
+++ b/core/src/test/java/org/apache/calcite/test/Matchers.java
@@ -16,10 +16,16 @@
*/
package org.apache.calcite.test;
+import org.apache.calcite.util.Util;
+
+import com.google.common.base.Functions;
+import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
+import org.hamcrest.BaseMatcher;
import org.hamcrest.CustomTypeSafeMatcher;
import org.hamcrest.Description;
+import org.hamcrest.Factory;
import org.hamcrest.Matcher;
import java.sql.ResultSet;
@@ -79,6 +85,81 @@ public class Matchers {
}
};
}
+
+ public static <E extends Comparable> Matcher<Iterable<E>> equalsUnordered(
+ E... lines) {
+ final List<String> expectedList =
+ Lists.newArrayList(toStringList(Arrays.asList(lines)));
+ Collections.sort(expectedList);
+ final String description = Util.lines(expectedList);
+ return new CustomTypeSafeMatcher<Iterable<E>>(description) {
+ @Override protected void describeMismatchSafely(Iterable<E> actuals,
+ Description description) {
+ final List<String> actualList =
+ Lists.newArrayList(toStringList(actuals));
+ Collections.sort(actualList);
+ description.appendText("was ")
+ .appendValue(Util.lines(actualList));
+ }
+
+ protected boolean matchesSafely(Iterable<E> actuals) {
+ final List<String> actualList =
+ Lists.newArrayList(toStringList(actuals));
+ Collections.sort(actualList);
+ return actualList.equals(expectedList);
+ }
+ };
+ }
+
+ private static <E> Iterable<String> toStringList(Iterable<E> items) {
+ return Iterables.transform(items, Functions.toStringFunction());
+ }
+
+ /**
+ * Creates a matcher that matches when the examined object is within
+ * {@code epsilon} of the specified <code>operand</code>.
+ */
+ @Factory
+ public static <T extends Number> Matcher<T> within(T value, double epsilon) {
+ return new IsWithin<T>(value, epsilon);
+ }
+
+ /**
+ * Is the numeric value within a given difference another value?
+ *
+ * @param <T> Value type
+ */
+ public static class IsWithin<T extends Number> extends BaseMatcher<T> {
+ private final T expectedValue;
+ private final double epsilon;
+
+ public IsWithin(T expectedValue, double epsilon) {
+ this.expectedValue = expectedValue;
+ this.epsilon = epsilon;
+ }
+
+ public boolean matches(Object actualValue) {
+ return areEqual(actualValue, expectedValue, epsilon);
+ }
+
+ public void describeTo(Description description) {
+ description.appendValue(expectedValue + " +/-" + epsilon);
+ }
+
+ private static boolean areEqual(Object actual, Number expected,
+ double epsilon) {
+ if (actual == null) {
+ return expected == null;
+ }
+ if (actual.equals(expected)) {
+ return true;
+ }
+ final double a = ((Number) actual).doubleValue();
+ final double min = expected.doubleValue() - epsilon;
+ final double max = expected.doubleValue() + epsilon;
+ return min <= a && a <= max;
+ }
+ }
}
// End Matchers.java
http://git-wip-us.apache.org/repos/asf/calcite/blob/dad58186/core/src/test/java/org/apache/calcite/util/PartiallyOrderedSetTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/calcite/util/PartiallyOrderedSetTest.java b/core/src/test/java/org/apache/calcite/util/PartiallyOrderedSetTest.java
index d37c223..8a6a9da 100644
--- a/core/src/test/java/org/apache/calcite/util/PartiallyOrderedSetTest.java
+++ b/core/src/test/java/org/apache/calcite/util/PartiallyOrderedSetTest.java
@@ -18,9 +18,13 @@ package org.apache.calcite.util;
import org.apache.calcite.test.CalciteAssert;
+import com.google.common.base.Function;
+
+import org.junit.Assume;
import org.junit.Test;
import java.util.AbstractList;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashSet;
import java.util.List;
@@ -28,7 +32,10 @@ import java.util.Random;
import java.util.Set;
import java.util.TreeSet;
+import static org.hamcrest.CoreMatchers.nullValue;
+import static org.hamcrest.core.Is.is;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
/**
@@ -133,6 +140,13 @@ public class PartiallyOrderedSetTest {
// "bcd" is child of "abcd" and parent of ""
final String bcd = "'bcd'";
+ assertEquals("['abcd']", poset.getParents(bcd, true).toString());
+ assertThat(poset.getParents(bcd, false), nullValue());
+ assertThat(poset.getParents(bcd), nullValue());
+ assertEquals("['']", poset.getChildren(bcd, true).toString());
+ assertThat(poset.getChildren(bcd, false), nullValue());
+ assertThat(poset.getChildren(bcd), nullValue());
+
poset.add(bcd);
printValidate(poset);
assertTrue(poset.isValid(false));
@@ -210,6 +224,66 @@ public class PartiallyOrderedSetTest {
printValidate(poset);
}
+ @Test public void testPosetBitsLarge() {
+ final PartiallyOrderedSet<Integer> poset =
+ new PartiallyOrderedSet<>(IS_BIT_SUPERSET);
+ checkPosetBitsLarge(poset, 30000, 2921, 164782);
+ }
+
+ @Test public void testPosetBitsLarge2() {
+ Assume.assumeTrue("too slow to run every day", CalciteAssert.ENABLE_SLOW);
+ final int n = 30000;
+ final PartiallyOrderedSet<Integer> poset =
+ new PartiallyOrderedSet<>(IS_BIT_SUPERSET,
+ new Function<Integer, Iterable<Integer>>() {
+ public Iterable<Integer> apply(Integer input) {
+ final int i = input;
+ int r = i; // bits not yet cleared
+ final List<Integer> list = new ArrayList<>();
+ for (int z = 1; r != 0; z <<= 1) {
+ if ((i & z) != 0) {
+ list.add(i ^ z);
+ r ^= z;
+ }
+ }
+ return list;
+ }
+ },
+ new Function<Integer, Iterable<Integer>>() {
+ public Iterable<Integer> apply(Integer input) {
+ final int i = input;
+ final List<Integer> list = new ArrayList<>();
+ for (int z = 1; z <= n; z <<= 1) {
+ if ((i & z) == 0) {
+ list.add(i | z);
+ }
+ }
+ return list;
+ }
+ });
+ checkPosetBitsLarge(poset, n, 2921, 11961);
+ }
+
+ void checkPosetBitsLarge(PartiallyOrderedSet<Integer> poset, int n,
+ int expectedSize, int expectedParentCount) {
+ final Random random = new Random(1);
+ int count = 0;
+ int parentCount = 0;
+ for (int i = 0; i < n; i++) {
+ if (random.nextInt(10) == 0) {
+ if (poset.add(random.nextInt(n * 2))) {
+ ++count;
+ }
+ }
+ final List<Integer> parents =
+ poset.getParents(random.nextInt(n * 2), true);
+ parentCount += parents.size();
+ }
+ assertThat(poset.size(), is(count));
+ assertThat(poset.size(), is(expectedSize));
+ assertThat(parentCount, is(expectedParentCount));
+ }
+
@Test public void testPosetBitsRemoveParent() {
final PartiallyOrderedSet<Integer> poset =
new PartiallyOrderedSet<Integer>(IS_BIT_SUPERSET);
@@ -223,9 +297,6 @@ public class PartiallyOrderedSetTest {
}
@Test public void testDivisorPoset() {
- if (!CalciteAssert.ENABLE_SLOW) {
- return;
- }
PartiallyOrderedSet<Integer> integers =
new PartiallyOrderedSet<Integer>(IS_DIVISOR, range(1, 1000));
assertEquals(
@@ -405,6 +476,7 @@ public class PartiallyOrderedSetTest {
expected,
new TreeSet<String>(ss).toString());
}
+
}
// End PartiallyOrderedSetTest.java
http://git-wip-us.apache.org/repos/asf/calcite/blob/dad58186/core/src/test/java/org/apache/calcite/util/TestUtil.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/calcite/util/TestUtil.java b/core/src/test/java/org/apache/calcite/util/TestUtil.java
index 5a283d2..b530d3b 100644
--- a/core/src/test/java/org/apache/calcite/util/TestUtil.java
+++ b/core/src/test/java/org/apache/calcite/util/TestUtil.java
@@ -34,6 +34,9 @@ public abstract class TestUtil {
private static final String LINE_BREAK =
"\\\\n\"" + Util.LINE_SEPARATOR + " + \"";
+ private static final String JAVA_VERSION =
+ System.getProperties().getProperty("java.version");
+
//~ Methods ----------------------------------------------------------------
public static void assertEqualsVerbose(
@@ -190,6 +193,20 @@ public abstract class TestUtil {
.replaceAll("\\[", "\\\\[")
.replaceAll("\\]", "\\\\]");
}
+
+ /** Returns the Java major version: 7 for JDK 1.7, 8 for JDK 8, 10 for
+ * JDK 10, etc. */
+ public static int getJavaMajorVersion() {
+ if (JAVA_VERSION.startsWith("1.7")) {
+ return 7;
+ } else if (JAVA_VERSION.startsWith("1.8")) {
+ return 8;
+ } else if (JAVA_VERSION.startsWith("1.9")) {
+ return 9;
+ } else {
+ return 10;
+ }
+ }
}
// End TestUtil.java
http://git-wip-us.apache.org/repos/asf/calcite/blob/dad58186/plus/src/test/java/org/apache/calcite/adapter/tpch/TpchTest.java
----------------------------------------------------------------------
diff --git a/plus/src/test/java/org/apache/calcite/adapter/tpch/TpchTest.java b/plus/src/test/java/org/apache/calcite/adapter/tpch/TpchTest.java
index 7dee05d..eaf9c91 100644
--- a/plus/src/test/java/org/apache/calcite/adapter/tpch/TpchTest.java
+++ b/plus/src/test/java/org/apache/calcite/adapter/tpch/TpchTest.java
@@ -19,6 +19,7 @@ package org.apache.calcite.adapter.tpch;
import org.apache.calcite.plan.RelOptUtil;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.test.CalciteAssert;
+import org.apache.calcite.util.TestUtil;
import org.apache.calcite.util.Util;
import com.google.common.base.Function;
@@ -40,11 +41,8 @@ import static org.junit.Assert.assertThat;
* if {@code -Dcalcite.test.slow} is specified on the command-line.
* (See {@link org.apache.calcite.test.CalciteAssert#ENABLE_SLOW}.)</p> */
public class TpchTest {
- public static final String JAVA_VERSION =
- System.getProperties().getProperty("java.version");
-
public static final boolean ENABLE =
- CalciteAssert.ENABLE_SLOW && JAVA_VERSION.compareTo("1.7") >= 0;
+ CalciteAssert.ENABLE_SLOW && TestUtil.getJavaMajorVersion() >= 7;
private static String schema(String name, String scaleFactor) {
return " {\n"
http://git-wip-us.apache.org/repos/asf/calcite/blob/dad58186/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 1feec71..9ea9ecc 100644
--- a/pom.xml
+++ b/pom.xml
@@ -129,6 +129,7 @@ limitations under the License.
<sqlline.version>1.3.0</sqlline.version>
<xalan.version>2.7.1</xalan.version>
<xerces.version>2.9.1</xerces.version>
+ <sketches.version>0.9.0</sketches.version>
</properties>
<issueManagement>
@@ -259,6 +260,11 @@ limitations under the License.
<version>${guava.version}</version>
</dependency>
<dependency>
+ <groupId>com.h2database</groupId>
+ <artifactId>h2</artifactId>
+ <version>${h2.version}</version>
+ </dependency>
+ <dependency>
<groupId>com.joestelmach</groupId>
<artifactId>natty</artifactId>
<version>${natty.version}</version>
@@ -269,9 +275,9 @@ limitations under the License.
<version>${oracle-jdbc6-driver.version}</version>
</dependency>
<dependency>
- <groupId>com.h2database</groupId>
- <artifactId>h2</artifactId>
- <version>${h2.version}</version>
+ <groupId>com.yahoo.datasketches</groupId>
+ <artifactId>sketches-core</artifactId>
+ <version>${sketches.version}</version>
</dependency>
<dependency>
<groupId>javax.servlet</groupId>