You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@calcite.apache.org by jh...@apache.org on 2015/09/03 00:16:10 UTC
[22/50] incubator-calcite git commit: [CALCITE-761] Pre-populated
materializations (Maryann Xue)
[CALCITE-761] Pre-populated materializations (Maryann Xue)
Project: http://git-wip-us.apache.org/repos/asf/incubator-calcite/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-calcite/commit/4b60b9bf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-calcite/tree/4b60b9bf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-calcite/diff/4b60b9bf
Branch: refs/heads/branch-release
Commit: 4b60b9bf46b88b4dc6af8279013cbb731d98a8d3
Parents: 629a56e
Author: maryannxue <we...@intel.com>
Authored: Fri Jun 26 15:32:02 2015 -0700
Committer: Julian Hyde <jh...@apache.org>
Committed: Tue Jul 21 16:51:15 2015 -0700
----------------------------------------------------------------------
.../materialize/MaterializationService.java | 19 +++++--
.../org/apache/calcite/model/ModelHandler.java | 17 +++++-
.../calcite/plan/RelOptMaterialization.java | 3 +-
.../schema/impl/MaterializedViewTable.java | 13 +++--
.../org/apache/calcite/test/CalciteAssert.java | 58 ++++++++++----------
.../calcite/test/MaterializationTest.java | 39 +++++++++++++
site/_docs/model.md | 7 ++-
7 files changed, 111 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/4b60b9bf/core/src/main/java/org/apache/calcite/materialize/MaterializationService.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/materialize/MaterializationService.java b/core/src/main/java/org/apache/calcite/materialize/MaterializationService.java
index b594cca..615d185 100644
--- a/core/src/main/java/org/apache/calcite/materialize/MaterializationService.java
+++ b/core/src/main/java/org/apache/calcite/materialize/MaterializationService.java
@@ -95,15 +95,16 @@ public class MaterializationService {
/** Defines a new materialization. Returns its key. */
public MaterializationKey defineMaterialization(final CalciteSchema schema,
TileKey tileKey, String viewSql, List<String> viewSchemaPath,
- final String suggestedTableName, boolean create) {
+ final String suggestedTableName, boolean create, boolean existing) {
return defineMaterialization(schema, tileKey, viewSql, viewSchemaPath,
- suggestedTableName, tableFactory, create);
+ suggestedTableName, tableFactory, create, existing);
}
/** Defines a new materialization. Returns its key. */
public MaterializationKey defineMaterialization(final CalciteSchema schema,
TileKey tileKey, String viewSql, List<String> viewSchemaPath,
- String suggestedTableName, TableFactory tableFactory, boolean create) {
+ String suggestedTableName, TableFactory tableFactory, boolean create,
+ boolean existing) {
final MaterializationActor.QueryKey queryKey =
new MaterializationActor.QueryKey(viewSql, schema, viewSchemaPath);
final MaterializationKey existingKey = actor.keyBySql.get(queryKey);
@@ -116,7 +117,15 @@ public class MaterializationService {
final CalciteConnection connection =
CalciteMetaImpl.connect(schema.root(), null);
- CalciteSchema.TableEntry tableEntry = schema.getTableBySql(viewSql);
+ CalciteSchema.TableEntry tableEntry;
+ if (existing) {
+ tableEntry = schema.getTable(suggestedTableName, true);
+ } else {
+ tableEntry = null;
+ }
+ if (tableEntry == null) {
+ tableEntry = schema.getTableBySql(viewSql);
+ }
RelDataType rowType = null;
if (tableEntry == null) {
Table table = tableFactory.createTable(schema, viewSql, viewSchemaPath);
@@ -265,7 +274,7 @@ public class MaterializationService {
final String sql = lattice.sql(groupSet, newTileKey.measures);
materializationKey =
defineMaterialization(schema, newTileKey, sql, schema.path(null),
- suggestedTableName, tableFactory, true);
+ suggestedTableName, tableFactory, true, false);
if (materializationKey != null) {
final CalciteSchema.TableEntry tableEntry =
checkValid(materializationKey);
http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/4b60b9bf/core/src/main/java/org/apache/calcite/model/ModelHandler.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/model/ModelHandler.java b/core/src/main/java/org/apache/calcite/model/ModelHandler.java
index 1afaf41..3faf446 100644
--- a/core/src/main/java/org/apache/calcite/model/ModelHandler.java
+++ b/core/src/main/java/org/apache/calcite/model/ModelHandler.java
@@ -265,9 +265,22 @@ public class ModelHandler {
+ "' is not a SemiMutableSchema");
}
CalciteSchema calciteSchema = CalciteSchema.from(schema);
- schema.add(jsonMaterialization.view,
+
+ final String viewName;
+ final boolean existing;
+ if (jsonMaterialization.view == null) {
+ // If the user did not supply a view name, that means the materialized
+ // view is pre-populated. Generate a synthetic view name.
+ viewName = "$" + schema.getTableNames().size();
+ existing = true;
+ } else {
+ viewName = jsonMaterialization.view;
+ existing = false;
+ }
+ schema.add(viewName,
MaterializedViewTable.create(calciteSchema,
- jsonMaterialization.getSql(), null, jsonMaterialization.table));
+ jsonMaterialization.getSql(), null, jsonMaterialization.table,
+ existing));
} catch (Exception e) {
throw new RuntimeException("Error instantiating " + jsonMaterialization,
e);
http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/4b60b9bf/core/src/main/java/org/apache/calcite/plan/RelOptMaterialization.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptMaterialization.java b/core/src/main/java/org/apache/calcite/plan/RelOptMaterialization.java
index f945e65..4108cff 100644
--- a/core/src/main/java/org/apache/calcite/plan/RelOptMaterialization.java
+++ b/core/src/main/java/org/apache/calcite/plan/RelOptMaterialization.java
@@ -62,7 +62,8 @@ public class RelOptMaterialization {
*/
public RelOptMaterialization(RelNode tableRel, RelNode queryRel,
RelOptTable starRelOptTable) {
- this.tableRel = tableRel;
+ this.tableRel =
+ RelOptUtil.createCastRel(tableRel, queryRel.getRowType(), false);
this.starRelOptTable = starRelOptTable;
if (starRelOptTable == null) {
this.starTable = null;
http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/4b60b9bf/core/src/main/java/org/apache/calcite/schema/impl/MaterializedViewTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/schema/impl/MaterializedViewTable.java b/core/src/main/java/org/apache/calcite/schema/impl/MaterializedViewTable.java
index 27e873a..8d80465 100644
--- a/core/src/main/java/org/apache/calcite/schema/impl/MaterializedViewTable.java
+++ b/core/src/main/java/org/apache/calcite/schema/impl/MaterializedViewTable.java
@@ -75,11 +75,10 @@ public class MaterializedViewTable extends ViewTable {
/** Table macro that returns a materialized view. */
public static MaterializedViewTableMacro create(final CalciteSchema schema,
- final String viewSql,
- final List<String> viewSchemaPath,
- final String tableName) {
+ final String viewSql, final List<String> viewSchemaPath,
+ final String suggestedTableName, boolean existing) {
return new MaterializedViewTableMacro(schema, viewSql, viewSchemaPath,
- tableName);
+ suggestedTableName, existing);
}
@Override public RelNode toRel(RelOptTable.ToRelContext context,
@@ -102,11 +101,13 @@ public class MaterializedViewTable extends ViewTable {
private final MaterializationKey key;
private MaterializedViewTableMacro(CalciteSchema schema, String viewSql,
- List<String> viewSchemaPath, String suggestedTableName) {
+ List<String> viewSchemaPath, String suggestedTableName,
+ boolean existing) {
super(schema, viewSql, viewSchemaPath, Boolean.TRUE);
this.key = Preconditions.checkNotNull(
MaterializationService.instance().defineMaterialization(
- schema, null, viewSql, schemaPath, suggestedTableName, true));
+ schema, null, viewSql, schemaPath, suggestedTableName, true,
+ existing));
}
@Override public TranslatableTable apply(List<Object> arguments) {
http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/4b60b9bf/core/src/test/java/org/apache/calcite/test/CalciteAssert.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/calcite/test/CalciteAssert.java b/core/src/test/java/org/apache/calcite/test/CalciteAssert.java
index 39ed982..3d0a0dc 100644
--- a/core/src/test/java/org/apache/calcite/test/CalciteAssert.java
+++ b/core/src/test/java/org/apache/calcite/test/CalciteAssert.java
@@ -405,7 +405,7 @@ public class CalciteAssert {
private static String typeString(ResultSetMetaData metaData)
throws SQLException {
- final List<String> list = new ArrayList<String>();
+ final List<String> list = new ArrayList<>();
for (int i = 0; i < metaData.getColumnCount(); i++) {
list.add(
metaData.getColumnName(i + 1)
@@ -452,13 +452,7 @@ public class CalciteAssert {
exceptionChecker.apply(null);
return;
}
- } catch (Exception e) {
- if (exceptionChecker != null) {
- exceptionChecker.apply(e);
- return;
- }
- throw e;
- } catch (Error e) {
+ } catch (Exception | Error e) {
if (exceptionChecker != null) {
exceptionChecker.apply(e);
return;
@@ -471,13 +465,11 @@ public class CalciteAssert {
resultSet.close();
statement.close();
connection.close();
- } catch (Error e) {
+ } catch (Error | RuntimeException e) {
// We ignore extended message for non-runtime exception, however
// it does not matter much since it is better to have AssertionError
// at the very top level of the exception stack.
throw e;
- } catch (RuntimeException e) {
- throw e;
} catch (Throwable e) {
throw new RuntimeException(message, e);
} finally {
@@ -840,21 +832,32 @@ public class CalciteAssert {
/** Adds materializations to the schema. */
public final AssertThat withMaterializations(String model,
- String... materializations) {
- assert materializations.length % 2 == 0;
+ final String... materializations) {
+ return withMaterializations(model,
+ new Function<JsonBuilder, List<Object>>() {
+ public List<Object> apply(JsonBuilder builder) {
+ assert materializations.length % 2 == 0;
+ final List<Object> list = builder.list();
+ for (int i = 0; i < materializations.length; i++) {
+ String table = materializations[i++];
+ final Map<String, Object> map = builder.map();
+ map.put("table", table);
+ map.put("view", table + "v");
+ String sql = materializations[i];
+ final String sql2 = sql.replaceAll("`", "\"");
+ map.put("sql", sql2);
+ list.add(map);
+ }
+ return list;
+ }
+ });
+ }
+
+ /** Adds materializations to the schema. */
+ public final AssertThat withMaterializations(String model,
+ Function<JsonBuilder, List<Object>> materializations) {
final JsonBuilder builder = new JsonBuilder();
- final List<Object> list = builder.list();
- for (int i = 0; i < materializations.length; i++) {
- String table = materializations[i++];
- final Map<String, Object> map = builder.map();
- map.put("table", table);
- map.put("view", table + "v");
- String sql = materializations[i];
- final String sql2 = sql
- .replaceAll("`", "\"");
- map.put("sql", sql2);
- list.add(map);
- }
+ final List<Object> list = materializations.apply(builder);
final String buf =
"materializations: " + builder.toJsonString(list);
final String model2;
@@ -908,13 +911,10 @@ public class CalciteAssert {
* and executes a callback. */
public <T> AssertThat doWithConnection(Function<CalciteConnection, T> fn)
throws Exception {
- Connection connection = connectionFactory.createConnection();
- try {
+ try (Connection connection = connectionFactory.createConnection()) {
T t = fn.apply((CalciteConnection) connection);
Util.discard(t);
return AssertThat.this;
- } finally {
- connection.close();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/4b60b9bf/core/src/test/java/org/apache/calcite/test/MaterializationTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/calcite/test/MaterializationTest.java b/core/src/test/java/org/apache/calcite/test/MaterializationTest.java
index ae235ab..f132915 100644
--- a/core/src/test/java/org/apache/calcite/test/MaterializationTest.java
+++ b/core/src/test/java/org/apache/calcite/test/MaterializationTest.java
@@ -28,14 +28,18 @@ import org.apache.calcite.rex.RexLiteral;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexUtil;
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.util.JsonBuilder;
import com.google.common.base.Function;
+import com.google.common.collect.ImmutableList;
import org.junit.Ignore;
import org.junit.Test;
import java.math.BigDecimal;
import java.sql.ResultSet;
+import java.util.List;
+import java.util.Map;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.junit.Assert.assertEquals;
@@ -53,6 +57,10 @@ public class MaterializationTest {
CalciteAssert.checkResultContains(
"EnumerableTableScan(table=[[hr, m0]])");
+ private static final Function<ResultSet, Void> CONTAINS_LOCATIONS =
+ CalciteAssert.checkResultContains(
+ "EnumerableTableScan(table=[[hr, locations]])");
+
final JavaTypeFactoryImpl typeFactory =
new JavaTypeFactoryImpl(RelDataTypeSystem.DEFAULT);
final RexBuilder rexBuilder = new RexBuilder(typeFactory);
@@ -749,6 +757,37 @@ public class MaterializationTest {
+ "join \"depts\" using (\"deptno\")";
checkMaterialize("select * from \"emps\" where \"empid\" < 500", q);
}
+
+ /** Test case for
+ * <a href="https://issues.apache.org/jira/browse/CALCITE-761">[CALCITE-761]
+ * Pre-populated materializations</a>. */
+ @Test public void testPrePopulated() {
+ String q = "select \"deptno\" from \"emps\"";
+ try {
+ Prepare.THREAD_TRIM.set(true);
+ MaterializationService.setThreadLocal();
+ CalciteAssert.that()
+ .withMaterializations(
+ JdbcTest.HR_MODEL,
+ new Function<JsonBuilder, List<Object>>() {
+ public List<Object> apply(JsonBuilder builder) {
+ final Map<String, Object> map = builder.map();
+ map.put("table", "locations");
+ String sql = "select `deptno` as `empid`, '' as `name`\n"
+ + "from `emps`";
+ final String sql2 = sql.replaceAll("`", "\"");
+ map.put("sql", sql2);
+ return ImmutableList.<Object>of(map);
+ }
+ })
+ .query(q)
+ .enableMaterializations(true)
+ .explainMatches("", CONTAINS_LOCATIONS)
+ .sameResultWithMaterializationsDisabled();
+ } finally {
+ Prepare.THREAD_TRIM.set(false);
+ }
+ }
}
// End MaterializationTest.java
http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/4b60b9bf/site/_docs/model.md
----------------------------------------------------------------------
diff --git a/site/_docs/model.md b/site/_docs/model.md
index 0f9a548..cb8cc9c 100644
--- a/site/_docs/model.md
+++ b/site/_docs/model.md
@@ -204,9 +204,12 @@ Occurs within `root.schemas.materializations`.
}
{% endhighlight %}
-`view` (optional string) TODO
+`view` (optional string) is the name of the view; null means that the table
+already exists and is populated with the correct data.
-`table` (optional string) TODO
+`table` (required string) is the name of the table that materializes the data in
+the query. If `view` is not null, the table might not exist, and if it does not,
+Calcite will create and populate an in-memory table.
`sql` (optional string, or list of strings that will be concatenated as a
multi-line string) is the SQL definition of the materialization.