You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@calcite.apache.org by GitBox <gi...@apache.org> on 2021/12/19 00:10:17 UTC
[GitHub] [calcite] jacques-n opened a new pull request #2653: [CALCITE-4952] (Re)introduce a simplistic RelMetadataQuery option
jacques-n opened a new pull request #2653:
URL: https://github.com/apache/calcite/pull/2653
- Introduce new ProxyingMetadataHandlerProvider that avoids code compilation
- Update RelMetadataTest to factor out metadata handling from tests
- Introduce tests for both Janino and the proxying/simplistic path.
- A simple microbenchmark for metadata
- Minor version bump to JMH
- Enable clearing of static cache within JaninoRelMetadataProvider
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@calcite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [calcite] jacques-n merged pull request #2653: [CALCITE-4952] (Re)introduce a simplistic RelMetadataQuery option
Posted by GitBox <gi...@apache.org>.
jacques-n merged pull request #2653:
URL: https://github.com/apache/calcite/pull/2653
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@calcite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [calcite] jacques-n commented on a change in pull request #2653: [CALCITE-4952] (Re)introduce a simplistic RelMetadataQuery option
Posted by GitBox <gi...@apache.org>.
jacques-n commented on a change in pull request #2653:
URL: https://github.com/apache/calcite/pull/2653#discussion_r774076758
##########
File path: ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/MetadataBenchmark.java
##########
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.benchmarks;
+
+import org.apache.calcite.jdbc.Driver;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.metadata.DefaultRelMetadataProvider;
+import org.apache.calcite.rel.metadata.JaninoRelMetadataProvider;
+import org.apache.calcite.rel.metadata.ProxyingMetadataHandlerProvider;
+import org.apache.calcite.rel.metadata.RelMetadataQuery;
+import org.apache.calcite.runtime.Hook;
+import org.apache.calcite.test.CalciteAssert;
+
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
+import java.util.function.Supplier;
+
+/**
+ * A benchmark to compare metadata retrieval time for a complex query.
+ *
+ * Compares metadata retrieval performance on a large query
+ *
+ */
+@Fork(value = 1, jvmArgsPrepend = "-Xmx2048m")
+@State(Scope.Benchmark)
+@Measurement(iterations = 10, time = 100, timeUnit = TimeUnit.MILLISECONDS)
+@Warmup(iterations = 10, time = 100, timeUnit = TimeUnit.MILLISECONDS)
+@Threads(1)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@BenchmarkMode(Mode.AverageTime)
+public class MetadataBenchmark {
+
+ @Setup
+ public void setup() throws SQLException {
+ DriverManager.registerDriver(new Driver());
+ }
+
+ private void test(final Supplier<RelMetadataQuery> supplier) {
+ CalciteAssert.that()
+ .with(CalciteAssert.Config.FOODMART_CLONE)
+ .query("select \"store\".\"store_country\" as \"c0\",\n"
+ + " \"time_by_day\".\"the_year\" as \"c1\",\n"
+ + " \"product_class\".\"product_family\" as \"c2\",\n"
+ + " count(\"sales_fact_1997\".\"product_id\") as \"m0\"\n"
+ + "from \"store\" as \"store\",\n"
+ + " \"sales_fact_1997\" as \"sales_fact_1997\",\n"
+ + " \"time_by_day\" as \"time_by_day\",\n"
+ + " \"product_class\" as \"product_class\",\n"
+ + " \"product\" as \"product\"\n"
+ + "where \"sales_fact_1997\".\"store_id\" = \"store\".\"store_id\"\n"
+ + "and \"store\".\"store_country\" = 'USA'\n"
+ + "and \"sales_fact_1997\".\"time_id\" = \"time_by_day\".\"time_id\"\n"
+ + "and \"time_by_day\".\"the_year\" = 1997\n"
+ + "and \"sales_fact_1997\".\"product_id\" = \"product\".\"product_id\"\n"
+ + "and \"product\".\"product_class_id\" = \"product_class\".\"product_class_id\"\n"
+ + "group by \"store\".\"store_country\",\n"
+ + " \"time_by_day\".\"the_year\",\n"
+ + " \"product_class\".\"product_family\"")
+ .withHook(Hook.CONVERTED, (Consumer<RelNode>) rel -> {
+ rel.getCluster().setMetadataQuerySupplier(supplier);
+ rel.getCluster().invalidateMetadataQuery();
+ })
+ .explainContains(""
+ + "EnumerableAggregate(group=[{1, 6, 10}], m0=[COUNT()])\n"
+ + " EnumerableMergeJoin(condition=[=($2, $8)], joinType=[inner])\n"
+ + " EnumerableSort(sort0=[$2], dir0=[ASC])\n"
+ + " EnumerableMergeJoin(condition=[=($3, $5)], joinType=[inner])\n"
+ + " EnumerableSort(sort0=[$3], dir0=[ASC])\n"
+ + " EnumerableHashJoin(condition=[=($0, $4)], joinType=[inner])\n"
+ + " EnumerableCalc(expr#0..23=[{inputs}], expr#24=['USA':VARCHAR(30)], "
+ + "expr#25=[=($t9, $t24)], store_id=[$t0], store_country=[$t9], $condition=[$t25])\n"
+ + " EnumerableTableScan(table=[[foodmart2, store]])\n"
+ + " EnumerableCalc(expr#0..7=[{inputs}], proj#0..1=[{exprs}], "
+ + "store_id=[$t4])\n"
+ + " EnumerableTableScan(table=[[foodmart2, sales_fact_1997]])\n"
+ + " EnumerableCalc(expr#0..9=[{inputs}], expr#10=[CAST($t4):INTEGER], "
+ + "expr#11=[1997], expr#12=[=($t10, $t11)], time_id=[$t0], the_year=[$t4], "
+ + "$condition=[$t12])\n"
+ + " EnumerableTableScan(table=[[foodmart2, time_by_day]])\n"
+ + " EnumerableHashJoin(condition=[=($0, $2)], joinType=[inner])\n"
+ + " EnumerableCalc(expr#0..14=[{inputs}], proj#0..1=[{exprs}])\n"
+ + " EnumerableTableScan(table=[[foodmart2, product]])\n"
+ + " EnumerableCalc(expr#0..4=[{inputs}], product_class_id=[$t0], "
+ + "product_family=[$t4])\n"
+ + " EnumerableTableScan(table=[[foodmart2, product_class]])")
+ .returns("c0=USA; c1=1997; c2=Non-Consumable; m0=16414\n"
+ + "c0=USA; c1=1997; c2=Drink; m0=7978\n"
+ + "c0=USA; c1=1997; c2=Food; m0=62445\n");
Review comment:
I agree in theory. In reality, I'm looking for baby steps here.
Based on my reviewing of metadata tickets back to the original refactor Julian did several years ago, it appears that multiple developers of the metadata module would manually use this query to test the performance. I wanted to at least get it in a place where it could be run in a reliable/standard/consistent way. Realistically, we should move to something that is actually a microbenchmark. Having the entirety of query planning rule firing/transformations/etc being part of a test to understand metadata performance leads to a lot of noise in the measurement (beyond just the plan validation).
For now, I thought it was better to keep this in since it also serves to confirm that the providers are behaving consistently.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@calcite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [calcite] jacques-n commented on a change in pull request #2653: [CALCITE-4952] (Re)introduce a simplistic RelMetadataQuery option
Posted by GitBox <gi...@apache.org>.
jacques-n commented on a change in pull request #2653:
URL: https://github.com/apache/calcite/pull/2653#discussion_r775619205
##########
File path: core/src/test/java/org/apache/calcite/test/RelMetadataTest.java
##########
@@ -3686,4 +4157,71 @@ Sql assertRowsUnique(boolean[] ignoreNulls, Matcher<Boolean> matcher,
}
}
+
+ /**
+ * A configuration that describes how metadata should be configured.
+ */
+ public static class MetadataConfig {
+
+ private final String name;
+ private final Function<RelMetadataProvider, MetadataHandlerProvider> converter;
+ private final Supplier<RelMetadataProvider> defaultProviderSupplier;
+ private final boolean isCaching;
Review comment:
I've just added a test to verify no cycles in both impls.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@calcite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [calcite] jacques-n commented on a change in pull request #2653: [CALCITE-4952] (Re)introduce a simplistic RelMetadataQuery option
Posted by GitBox <gi...@apache.org>.
jacques-n commented on a change in pull request #2653:
URL: https://github.com/apache/calcite/pull/2653#discussion_r773442182
##########
File path: core/src/test/java/org/apache/calcite/test/RelMetadataTest.java
##########
@@ -3686,4 +4157,71 @@ Sql assertRowsUnique(boolean[] ignoreNulls, Matcher<Boolean> matcher,
}
}
+
+ /**
+ * A configuration that describes how metadata should be configured.
+ */
+ public static class MetadataConfig {
+
+ private final String name;
+ private final Function<RelMetadataProvider, MetadataHandlerProvider> converter;
+ private final Supplier<RelMetadataProvider> defaultProviderSupplier;
+ private final boolean isCaching;
Review comment:
I'm wondering if you're referring of the requirement to ensure we're breaking cycles (as opposed to actually caching). Do you know if there are any tests to verify this behavior (cycle-break)?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@calcite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [calcite] zabetak commented on a change in pull request #2653: [CALCITE-4952] (Re)introduce a simplistic RelMetadataQuery option
Posted by GitBox <gi...@apache.org>.
zabetak commented on a change in pull request #2653:
URL: https://github.com/apache/calcite/pull/2653#discussion_r774121945
##########
File path: core/src/main/java/org/apache/calcite/rel/metadata/ProxyingMetadataHandlerProvider.java
##########
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.rel.metadata;
+
+import org.apache.calcite.rel.RelNode;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.Method;
+import java.lang.reflect.ParameterizedType;
+import java.lang.reflect.Proxy;
+import java.lang.reflect.Type;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+/**
+ * A MetadataHandlerProvider built on a RelMetadataProvider.
+ *
+ * Uses proxies to call the underlying metadata provider.
+ */
+public class ProxyingMetadataHandlerProvider implements MetadataHandlerProvider {
+
+ private final RelMetadataProvider provider;
+
+ /**
+ * Create a proxying handler provider.
+ * @param provider The provider this will operate against.
+ */
+ public ProxyingMetadataHandlerProvider(RelMetadataProvider provider) {
+ this.provider = provider;
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override public <MH extends MetadataHandler<?>> MH handler(Class<MH> handlerClass) {
+
+ Type[] types = handlerClass.getGenericInterfaces();
+ if (types.length != 1 || !(types[0] instanceof ParameterizedType)) {
+ throw new UnsupportedOperationException("Unexpected failure. " + handlerClass);
+ }
+
+ ParameterizedType pType = (ParameterizedType) types[0];
+ if (pType.getRawType() != MetadataHandler.class) {
+ throw new UnsupportedOperationException("Unexpected failure. " + handlerClass);
+ }
+ Class<?> metadataType = (Class<?>) pType.getActualTypeArguments()[0];
+ final Field field;
+ final MetadataDef<?> def;
+ try {
+ field = metadataType.getField("DEF");
+ def = Objects.requireNonNull((MetadataDef<?>) field.get(null),
+ () -> "Unexpected failure. " + handlerClass);
+ } catch (NoSuchFieldException | IllegalAccessException e) {
+ throw new RuntimeException(e);
+ }
+
+ List<Method> methods = def.methods;
+ Map<String, Method> methodMap = methods.stream()
+ .collect(Collectors.toMap(Method::getName, f -> f));
+ InvocationHandler handler = (proxy, method, args) -> {
+ Method metadataMethod = Objects.requireNonNull(methodMap.get(method.getName()),
+ () -> "Not supported: " + method);
+ RelNode rel = Objects.requireNonNull((RelNode) args[0], "rel must be non null");
+ RelMetadataQuery mq = Objects.requireNonNull((RelMetadataQuery) args[1],
+ "mq must be non null");
+
+ // using deprecated RelMetadataProvider method here as the non-deprecated methods completely
+ // sidestep the purpose of RelMetadataProvider reflection-based functionality.
+ UnboundMetadata metadata = provider.apply(
+ (Class<? extends RelNode>) rel.getClass(),
+ (Class<? extends Metadata>) metadataType);
Review comment:
I didn't follow closely the changes in metadata APIs but personally I would refrain from deprecating/removing APIs when in doubt. Taking out a release with a new deprecation has an impact on end-users and we cannot change our mind very often. Having said that, this is not a discussion for this PR so feel free to move it forward as you see fit.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@calcite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [calcite] zabetak commented on a change in pull request #2653: [CALCITE-4952] (Re)introduce a simplistic RelMetadataQuery option
Posted by GitBox <gi...@apache.org>.
zabetak commented on a change in pull request #2653:
URL: https://github.com/apache/calcite/pull/2653#discussion_r772263213
##########
File path: core/src/test/java/org/apache/calcite/test/RelMetadataTest.java
##########
@@ -3686,4 +3742,43 @@ Sql assertRowsUnique(boolean[] ignoreNulls, Matcher<Boolean> matcher,
}
}
+
+ /**
+ * Test class for janino metadata path.
+ */
+ @Nested
Review comment:
I am not sure if this is an appropriate use of the `@Nested` annotation. Citing the [JUnit5 user guide](https://junit.org/junit5/docs/current/user-guide/#writing-tests-nested):
`Only non-static nested classes (i.e. inner classes) can serve as @Nested test classes`.
I think the test run in this case cause you are using also inheritance but again I am not sure if those were meant to be combined. Instead of relying on test inheritance or `@Nested`, [parameterized tests](https://junit.org/junit5/docs/current/user-guide/#writing-tests-parameterized-tests) would better fit here I think.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@calcite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [calcite] jamesstarr commented on a change in pull request #2653: [CALCITE-4952] (Re)introduce a simplistic RelMetadataQuery option
Posted by GitBox <gi...@apache.org>.
jamesstarr commented on a change in pull request #2653:
URL: https://github.com/apache/calcite/pull/2653#discussion_r772700725
##########
File path: core/src/test/java/org/apache/calcite/test/RelMetadataTest.java
##########
@@ -3686,4 +4157,71 @@ Sql assertRowsUnique(boolean[] ignoreNulls, Matcher<Boolean> matcher,
}
}
+
+ /**
+ * A configuration that describes how metadata should be configured.
+ */
+ public static class MetadataConfig {
+
+ private final String name;
+ private final Function<RelMetadataProvider, MetadataHandlerProvider> converter;
+ private final Supplier<RelMetadataProvider> defaultProviderSupplier;
+ private final boolean isCaching;
Review comment:
Caching is required volcano planner.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@calcite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [calcite] zabetak commented on a change in pull request #2653: [CALCITE-4952] (Re)introduce a simplistic RelMetadataQuery option
Posted by GitBox <gi...@apache.org>.
zabetak commented on a change in pull request #2653:
URL: https://github.com/apache/calcite/pull/2653#discussion_r774125076
##########
File path: ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/MetadataBenchmark.java
##########
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.benchmarks;
+
+import org.apache.calcite.jdbc.Driver;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.metadata.DefaultRelMetadataProvider;
+import org.apache.calcite.rel.metadata.JaninoRelMetadataProvider;
+import org.apache.calcite.rel.metadata.ProxyingMetadataHandlerProvider;
+import org.apache.calcite.rel.metadata.RelMetadataQuery;
+import org.apache.calcite.runtime.Hook;
+import org.apache.calcite.test.CalciteAssert;
+
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
+import java.util.function.Supplier;
+
+/**
+ * A benchmark to compare metadata retrieval time for a complex query.
+ *
+ * Compares metadata retrieval performance on a large query
+ *
+ */
+@Fork(value = 1, jvmArgsPrepend = "-Xmx2048m")
+@State(Scope.Benchmark)
+@Measurement(iterations = 10, time = 100, timeUnit = TimeUnit.MILLISECONDS)
+@Warmup(iterations = 10, time = 100, timeUnit = TimeUnit.MILLISECONDS)
+@Threads(1)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@BenchmarkMode(Mode.AverageTime)
+public class MetadataBenchmark {
+
+ @Setup
+ public void setup() throws SQLException {
+ DriverManager.registerDriver(new Driver());
+ }
+
+ private void test(final Supplier<RelMetadataQuery> supplier) {
+ CalciteAssert.that()
+ .with(CalciteAssert.Config.FOODMART_CLONE)
+ .query("select \"store\".\"store_country\" as \"c0\",\n"
+ + " \"time_by_day\".\"the_year\" as \"c1\",\n"
+ + " \"product_class\".\"product_family\" as \"c2\",\n"
+ + " count(\"sales_fact_1997\".\"product_id\") as \"m0\"\n"
+ + "from \"store\" as \"store\",\n"
+ + " \"sales_fact_1997\" as \"sales_fact_1997\",\n"
+ + " \"time_by_day\" as \"time_by_day\",\n"
+ + " \"product_class\" as \"product_class\",\n"
+ + " \"product\" as \"product\"\n"
+ + "where \"sales_fact_1997\".\"store_id\" = \"store\".\"store_id\"\n"
+ + "and \"store\".\"store_country\" = 'USA'\n"
+ + "and \"sales_fact_1997\".\"time_id\" = \"time_by_day\".\"time_id\"\n"
+ + "and \"time_by_day\".\"the_year\" = 1997\n"
+ + "and \"sales_fact_1997\".\"product_id\" = \"product\".\"product_id\"\n"
+ + "and \"product\".\"product_class_id\" = \"product_class\".\"product_class_id\"\n"
+ + "group by \"store\".\"store_country\",\n"
+ + " \"time_by_day\".\"the_year\",\n"
+ + " \"product_class\".\"product_family\"")
+ .withHook(Hook.CONVERTED, (Consumer<RelNode>) rel -> {
+ rel.getCluster().setMetadataQuerySupplier(supplier);
+ rel.getCluster().invalidateMetadataQuery();
+ })
+ .explainContains(""
+ + "EnumerableAggregate(group=[{1, 6, 10}], m0=[COUNT()])\n"
+ + " EnumerableMergeJoin(condition=[=($2, $8)], joinType=[inner])\n"
+ + " EnumerableSort(sort0=[$2], dir0=[ASC])\n"
+ + " EnumerableMergeJoin(condition=[=($3, $5)], joinType=[inner])\n"
+ + " EnumerableSort(sort0=[$3], dir0=[ASC])\n"
+ + " EnumerableHashJoin(condition=[=($0, $4)], joinType=[inner])\n"
+ + " EnumerableCalc(expr#0..23=[{inputs}], expr#24=['USA':VARCHAR(30)], "
+ + "expr#25=[=($t9, $t24)], store_id=[$t0], store_country=[$t9], $condition=[$t25])\n"
+ + " EnumerableTableScan(table=[[foodmart2, store]])\n"
+ + " EnumerableCalc(expr#0..7=[{inputs}], proj#0..1=[{exprs}], "
+ + "store_id=[$t4])\n"
+ + " EnumerableTableScan(table=[[foodmart2, sales_fact_1997]])\n"
+ + " EnumerableCalc(expr#0..9=[{inputs}], expr#10=[CAST($t4):INTEGER], "
+ + "expr#11=[1997], expr#12=[=($t10, $t11)], time_id=[$t0], the_year=[$t4], "
+ + "$condition=[$t12])\n"
+ + " EnumerableTableScan(table=[[foodmart2, time_by_day]])\n"
+ + " EnumerableHashJoin(condition=[=($0, $2)], joinType=[inner])\n"
+ + " EnumerableCalc(expr#0..14=[{inputs}], proj#0..1=[{exprs}])\n"
+ + " EnumerableTableScan(table=[[foodmart2, product]])\n"
+ + " EnumerableCalc(expr#0..4=[{inputs}], product_class_id=[$t0], "
+ + "product_family=[$t4])\n"
+ + " EnumerableTableScan(table=[[foodmart2, product_class]])")
+ .returns("c0=USA; c1=1997; c2=Non-Consumable; m0=16414\n"
+ + "c0=USA; c1=1997; c2=Drink; m0=7978\n"
+ + "c0=USA; c1=1997; c2=Food; m0=62445\n");
Review comment:
I am not saying to turn this into a real microbenchmark since we are quite far from there anyways. However, if we need to ensure correctness then wouldn't be better to add unit tests for this (if there are aren't already). Anyways minor comment feel free to ignore.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@calcite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [calcite] jacques-n commented on a change in pull request #2653: [CALCITE-4952] (Re)introduce a simplistic RelMetadataQuery option
Posted by GitBox <gi...@apache.org>.
jacques-n commented on a change in pull request #2653:
URL: https://github.com/apache/calcite/pull/2653#discussion_r772563991
##########
File path: core/src/test/java/org/apache/calcite/test/RelMetadataTest.java
##########
@@ -3686,4 +3742,43 @@ Sql assertRowsUnique(boolean[] ignoreNulls, Matcher<Boolean> matcher,
}
}
+
+ /**
+ * Test class for janino metadata path.
+ */
+ @Nested
Review comment:
Yeah, I was torn on making it parameterized given the sheer amount of minor changes.
I just went through and did so. It quadrupled the size of the patch but I agree it is more idiomatic.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@calcite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [calcite] jamesstarr commented on a change in pull request #2653: [CALCITE-4952] (Re)introduce a simplistic RelMetadataQuery option
Posted by GitBox <gi...@apache.org>.
jamesstarr commented on a change in pull request #2653:
URL: https://github.com/apache/calcite/pull/2653#discussion_r773482083
##########
File path: core/src/test/java/org/apache/calcite/test/RelMetadataTest.java
##########
@@ -3686,4 +4157,71 @@ Sql assertRowsUnique(boolean[] ignoreNulls, Matcher<Boolean> matcher,
}
}
+
+ /**
+ * A configuration that describes how metadata should be configured.
+ */
+ public static class MetadataConfig {
+
+ private final String name;
+ private final Function<RelMetadataProvider, MetadataHandlerProvider> converter;
+ private final Supplier<RelMetadataProvider> defaultProviderSupplier;
+ private final boolean isCaching;
Review comment:
Yes, I am refering to cycle detection. Sorry, about not being clearer. I am not sure if their are any tests in calcite that trigger it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@calcite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [calcite] zabetak commented on a change in pull request #2653: [CALCITE-4952] (Re)introduce a simplistic RelMetadataQuery option
Posted by GitBox <gi...@apache.org>.
zabetak commented on a change in pull request #2653:
URL: https://github.com/apache/calcite/pull/2653#discussion_r773749631
##########
File path: ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/MetadataBenchmark.java
##########
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.benchmarks;
+
+import org.apache.calcite.jdbc.Driver;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.metadata.DefaultRelMetadataProvider;
+import org.apache.calcite.rel.metadata.JaninoRelMetadataProvider;
+import org.apache.calcite.rel.metadata.ProxyingMetadataHandlerProvider;
+import org.apache.calcite.rel.metadata.RelMetadataQuery;
+import org.apache.calcite.runtime.Hook;
+import org.apache.calcite.test.CalciteAssert;
+
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
+import java.util.function.Supplier;
+
+/**
+ * A benchmark to compare metadata retrieval time for a complex query.
+ *
+ * Compares metadata retrieval performance on a large query
+ *
+ */
+@Fork(value = 1, jvmArgsPrepend = "-Xmx2048m")
+@State(Scope.Benchmark)
+@Measurement(iterations = 10, time = 100, timeUnit = TimeUnit.MILLISECONDS)
+@Warmup(iterations = 10, time = 100, timeUnit = TimeUnit.MILLISECONDS)
+@Threads(1)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@BenchmarkMode(Mode.AverageTime)
+public class MetadataBenchmark {
+
+ @Setup
+ public void setup() throws SQLException {
+ DriverManager.registerDriver(new Driver());
+ }
+
+ private void test(final Supplier<RelMetadataQuery> supplier) {
+ CalciteAssert.that()
+ .with(CalciteAssert.Config.FOODMART_CLONE)
+ .query("select \"store\".\"store_country\" as \"c0\",\n"
+ + " \"time_by_day\".\"the_year\" as \"c1\",\n"
+ + " \"product_class\".\"product_family\" as \"c2\",\n"
+ + " count(\"sales_fact_1997\".\"product_id\") as \"m0\"\n"
+ + "from \"store\" as \"store\",\n"
+ + " \"sales_fact_1997\" as \"sales_fact_1997\",\n"
+ + " \"time_by_day\" as \"time_by_day\",\n"
+ + " \"product_class\" as \"product_class\",\n"
+ + " \"product\" as \"product\"\n"
+ + "where \"sales_fact_1997\".\"store_id\" = \"store\".\"store_id\"\n"
+ + "and \"store\".\"store_country\" = 'USA'\n"
+ + "and \"sales_fact_1997\".\"time_id\" = \"time_by_day\".\"time_id\"\n"
+ + "and \"time_by_day\".\"the_year\" = 1997\n"
+ + "and \"sales_fact_1997\".\"product_id\" = \"product\".\"product_id\"\n"
+ + "and \"product\".\"product_class_id\" = \"product_class\".\"product_class_id\"\n"
+ + "group by \"store\".\"store_country\",\n"
+ + " \"time_by_day\".\"the_year\",\n"
+ + " \"product_class\".\"product_family\"")
+ .withHook(Hook.CONVERTED, (Consumer<RelNode>) rel -> {
+ rel.getCluster().setMetadataQuerySupplier(supplier);
+ rel.getCluster().invalidateMetadataQuery();
+ })
+ .explainContains(""
+ + "EnumerableAggregate(group=[{1, 6, 10}], m0=[COUNT()])\n"
+ + " EnumerableMergeJoin(condition=[=($2, $8)], joinType=[inner])\n"
+ + " EnumerableSort(sort0=[$2], dir0=[ASC])\n"
+ + " EnumerableMergeJoin(condition=[=($3, $5)], joinType=[inner])\n"
+ + " EnumerableSort(sort0=[$3], dir0=[ASC])\n"
+ + " EnumerableHashJoin(condition=[=($0, $4)], joinType=[inner])\n"
+ + " EnumerableCalc(expr#0..23=[{inputs}], expr#24=['USA':VARCHAR(30)], "
+ + "expr#25=[=($t9, $t24)], store_id=[$t0], store_country=[$t9], $condition=[$t25])\n"
+ + " EnumerableTableScan(table=[[foodmart2, store]])\n"
+ + " EnumerableCalc(expr#0..7=[{inputs}], proj#0..1=[{exprs}], "
+ + "store_id=[$t4])\n"
+ + " EnumerableTableScan(table=[[foodmart2, sales_fact_1997]])\n"
+ + " EnumerableCalc(expr#0..9=[{inputs}], expr#10=[CAST($t4):INTEGER], "
+ + "expr#11=[1997], expr#12=[=($t10, $t11)], time_id=[$t0], the_year=[$t4], "
+ + "$condition=[$t12])\n"
+ + " EnumerableTableScan(table=[[foodmart2, time_by_day]])\n"
+ + " EnumerableHashJoin(condition=[=($0, $2)], joinType=[inner])\n"
+ + " EnumerableCalc(expr#0..14=[{inputs}], proj#0..1=[{exprs}])\n"
+ + " EnumerableTableScan(table=[[foodmart2, product]])\n"
+ + " EnumerableCalc(expr#0..4=[{inputs}], product_class_id=[$t0], "
+ + "product_family=[$t4])\n"
+ + " EnumerableTableScan(table=[[foodmart2, product_class]])")
+ .returns("c0=USA; c1=1997; c2=Non-Consumable; m0=16414\n"
+ + "c0=USA; c1=1997; c2=Drink; m0=7978\n"
+ + "c0=USA; c1=1997; c2=Food; m0=62445\n");
Review comment:
Probably we don't care to measure this part. I guess it doesn't really matter cause you care about the relative difference and not the absolute numbers but removing unnecessary parts could improve the accuracy of the measurement.
##########
File path: core/src/main/java/org/apache/calcite/rel/metadata/ProxyingMetadataHandlerProvider.java
##########
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.rel.metadata;
+
+import org.apache.calcite.rel.RelNode;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.Method;
+import java.lang.reflect.ParameterizedType;
+import java.lang.reflect.Proxy;
+import java.lang.reflect.Type;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+/**
+ * A MetadataHandlerProvider built on a RelMetadataProvider.
+ *
+ * Uses proxies to call the underlying metadata provider.
+ */
+public class ProxyingMetadataHandlerProvider implements MetadataHandlerProvider {
+
+ private final RelMetadataProvider provider;
+
+ /**
+ * Create a proxying handler provider.
+ * @param provider The provider this will operate against.
+ */
+ public ProxyingMetadataHandlerProvider(RelMetadataProvider provider) {
+ this.provider = provider;
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override public <MH extends MetadataHandler<?>> MH handler(Class<MH> handlerClass) {
+
+ Type[] types = handlerClass.getGenericInterfaces();
+ if (types.length != 1 || !(types[0] instanceof ParameterizedType)) {
+ throw new UnsupportedOperationException("Unexpected failure. " + handlerClass);
+ }
+
+ ParameterizedType pType = (ParameterizedType) types[0];
+ if (pType.getRawType() != MetadataHandler.class) {
+ throw new UnsupportedOperationException("Unexpected failure. " + handlerClass);
+ }
+ Class<?> metadataType = (Class<?>) pType.getActualTypeArguments()[0];
+ final Field field;
+ final MetadataDef<?> def;
+ try {
+ field = metadataType.getField("DEF");
+ def = Objects.requireNonNull((MetadataDef<?>) field.get(null),
+ () -> "Unexpected failure. " + handlerClass);
+ } catch (NoSuchFieldException | IllegalAccessException e) {
+ throw new RuntimeException(e);
+ }
+
+ List<Method> methods = def.methods;
+ Map<String, Method> methodMap = methods.stream()
+ .collect(Collectors.toMap(Method::getName, f -> f));
+ InvocationHandler handler = (proxy, method, args) -> {
+ Method metadataMethod = Objects.requireNonNull(methodMap.get(method.getName()),
+ () -> "Not supported: " + method);
+ RelNode rel = Objects.requireNonNull((RelNode) args[0], "rel must be non null");
+ RelMetadataQuery mq = Objects.requireNonNull((RelMetadataQuery) args[1],
+ "mq must be non null");
+
+ // using deprecated RelMetadataProvider method here as the non-deprecated methods completely
+ // sidestep the purpose of RelMetadataProvider reflection-based functionality.
+ UnboundMetadata metadata = provider.apply(
+ (Class<? extends RelNode>) rel.getClass(),
+ (Class<? extends Metadata>) metadataType);
Review comment:
I don't think it is a good idea to use deprecated APIs. If we really need this method one way or another should we revert the deprecation?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@calcite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [calcite] jacques-n commented on a change in pull request #2653: [CALCITE-4952] (Re)introduce a simplistic RelMetadataQuery option
Posted by GitBox <gi...@apache.org>.
jacques-n commented on a change in pull request #2653:
URL: https://github.com/apache/calcite/pull/2653#discussion_r774078632
##########
File path: core/src/main/java/org/apache/calcite/rel/metadata/ProxyingMetadataHandlerProvider.java
##########
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.rel.metadata;
+
+import org.apache.calcite.rel.RelNode;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.Method;
+import java.lang.reflect.ParameterizedType;
+import java.lang.reflect.Proxy;
+import java.lang.reflect.Type;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+/**
+ * A MetadataHandlerProvider built on a RelMetadataProvider.
+ *
+ * Uses proxies to call the underlying metadata provider.
+ */
+public class ProxyingMetadataHandlerProvider implements MetadataHandlerProvider {
+
+ private final RelMetadataProvider provider;
+
+ /**
+ * Create a proxying handler provider.
+ * @param provider The provider this will operate against.
+ */
+ public ProxyingMetadataHandlerProvider(RelMetadataProvider provider) {
+ this.provider = provider;
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override public <MH extends MetadataHandler<?>> MH handler(Class<MH> handlerClass) {
+
+ Type[] types = handlerClass.getGenericInterfaces();
+ if (types.length != 1 || !(types[0] instanceof ParameterizedType)) {
+ throw new UnsupportedOperationException("Unexpected failure. " + handlerClass);
+ }
+
+ ParameterizedType pType = (ParameterizedType) types[0];
+ if (pType.getRawType() != MetadataHandler.class) {
+ throw new UnsupportedOperationException("Unexpected failure. " + handlerClass);
+ }
+ Class<?> metadataType = (Class<?>) pType.getActualTypeArguments()[0];
+ final Field field;
+ final MetadataDef<?> def;
+ try {
+ field = metadataType.getField("DEF");
+ def = Objects.requireNonNull((MetadataDef<?>) field.get(null),
+ () -> "Unexpected failure. " + handlerClass);
+ } catch (NoSuchFieldException | IllegalAccessException e) {
+ throw new RuntimeException(e);
+ }
+
+ List<Method> methods = def.methods;
+ Map<String, Method> methodMap = methods.stream()
+ .collect(Collectors.toMap(Method::getName, f -> f));
+ InvocationHandler handler = (proxy, method, args) -> {
+ Method metadataMethod = Objects.requireNonNull(methodMap.get(method.getName()),
+ () -> "Not supported: " + method);
+ RelNode rel = Objects.requireNonNull((RelNode) args[0], "rel must be non null");
+ RelMetadataQuery mq = Objects.requireNonNull((RelMetadataQuery) args[1],
+ "mq must be non null");
+
+ // using deprecated RelMetadataProvider method here as the non-deprecated methods completely
+ // sidestep the purpose of RelMetadataProvider reflection-based functionality.
+ UnboundMetadata metadata = provider.apply(
+ (Class<? extends RelNode>) rel.getClass(),
+ (Class<? extends Metadata>) metadataType);
Review comment:
This isn't my favorite either but I think it's the right choice for now. The apis are so much in flux at the moment that reverting deprecation isn't ideal since I'm not convinced that the deprecated interface is best either. (In a perfect world, we would have included the second impl before deciding to deprecate this method.) A risk associated with the ongoing refactor in this module is too much focus on a single impl (the janino one). It's important to have this second impl in place so we can make sure the refactoring makes sense across impls.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@calcite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [calcite] zabetak commented on a change in pull request #2653: [CALCITE-4952] (Re)introduce a simplistic RelMetadataQuery option
Posted by GitBox <gi...@apache.org>.
zabetak commented on a change in pull request #2653:
URL: https://github.com/apache/calcite/pull/2653#discussion_r773745327
##########
File path: core/src/test/java/org/apache/calcite/test/RelMetadataTest.java
##########
@@ -3686,4 +3742,43 @@ Sql assertRowsUnique(boolean[] ignoreNulls, Matcher<Boolean> matcher,
}
}
+
+ /**
+ * Test class for janino metadata path.
+ */
+ @Nested
Review comment:
Oh, sorry about that. It wasn't a "must do" comment from my side but thanks for taking care of it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@calcite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [calcite] jacques-n commented on a change in pull request #2653: [CALCITE-4952] (Re)introduce a simplistic RelMetadataQuery option
Posted by GitBox <gi...@apache.org>.
jacques-n commented on a change in pull request #2653:
URL: https://github.com/apache/calcite/pull/2653#discussion_r772705972
##########
File path: core/src/test/java/org/apache/calcite/test/RelMetadataTest.java
##########
@@ -3686,4 +4157,71 @@ Sql assertRowsUnique(boolean[] ignoreNulls, Matcher<Boolean> matcher,
}
}
+
+ /**
+ * A configuration that describes how metadata should be configured.
+ */
+ public static class MetadataConfig {
+
+ private final String name;
+ private final Function<RelMetadataProvider, MetadataHandlerProvider> converter;
+ private final Supplier<RelMetadataProvider> defaultProviderSupplier;
+ private final boolean isCaching;
Review comment:
Can you point me to where that is? That seems like it should be a bug in volcano, not a requirement.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@calcite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org