You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2018/10/09 15:50:28 UTC
[26/45] carbondata git commit: Problem: Preaggregate select queries
require table path of parent table to access dictionary files. Therefore in
executor CarbonMetadata class was used to get parent table object. As
CarbonMetadata class is only meant to be
Problem:
Preaggregate select queries require table path of parent table to access dictionary files. Therefore in executor CarbonMetadata class was used to get parent table object. As CarbonMetadata class is only meant to be used in driver and is not filled with carbontable objects for select queries therefore the query was throwing NPE.
Solution:
Pass parent table path from driver to executor by adding a new variable in RelationIdentifier. This will not be written to thrift, instead will be used to carry tablePath property from driver to executor.
This closes #2786
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/9ca985f0
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/9ca985f0
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/9ca985f0
Branch: refs/heads/branch-1.5
Commit: 9ca985f0d96380d96dae6fdab2d4ee014d5ac345
Parents: 682160f
Author: kunal642 <ku...@gmail.com>
Authored: Fri Sep 28 17:41:32 2018 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Wed Oct 3 20:02:49 2018 +0530
----------------------------------------------------------------------
.../core/metadata/schema/table/RelationIdentifier.java | 12 ++++++++++++
.../carbondata/core/scan/executor/util/QueryUtil.java | 10 ++--------
.../spark/sql/CarbonDatasourceHadoopRelation.scala | 12 ++++++++++++
3 files changed, 26 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9ca985f0/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/RelationIdentifier.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/RelationIdentifier.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/RelationIdentifier.java
index 9a1dad1..0e8042d 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/RelationIdentifier.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/RelationIdentifier.java
@@ -32,6 +32,8 @@ public class RelationIdentifier implements Serializable, Writable {
private String tableId;
+ private String tablePath = "";
+
public RelationIdentifier(String databaseName, String tableName, String tableId) {
this.databaseName = databaseName;
this.tableName = tableName;
@@ -50,16 +52,26 @@ public class RelationIdentifier implements Serializable, Writable {
return tableId;
}
+ public String getTablePath() {
+ return tablePath;
+ }
+
+ public void setTablePath(String tablePath) {
+ this.tablePath = tablePath;
+ }
+
@Override public void write(DataOutput out) throws IOException {
out.writeUTF(databaseName);
out.writeUTF(tableName);
out.writeUTF(tableId);
+ out.writeUTF(tablePath);
}
@Override public void readFields(DataInput in) throws IOException {
this.databaseName = in.readUTF();
this.tableName = in.readUTF();
this.tableId = in.readUTF();
+ this.tablePath = in.readUTF();
}
@Override public boolean equals(Object o) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9ca985f0/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
index 9fb0857..7849d10 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
@@ -393,12 +393,6 @@ public class QueryUtil {
public static AbsoluteTableIdentifier getTableIdentifierForColumn(
CarbonDimension carbonDimension) {
- RelationIdentifier parentRelationIdentifier =
- carbonDimension.getColumnSchema().getParentColumnTableRelations().get(0)
- .getRelationIdentifier();
- String parentTablePath = CarbonMetadata.getInstance()
- .getCarbonTable(parentRelationIdentifier.getDatabaseName(),
- parentRelationIdentifier.getTableName()).getTablePath();
RelationIdentifier relation = carbonDimension.getColumnSchema()
.getParentColumnTableRelations()
.get(0)
@@ -406,8 +400,8 @@ public class QueryUtil {
String parentTableName = relation.getTableName();
String parentDatabaseName = relation.getDatabaseName();
String parentTableId = relation.getTableId();
- return AbsoluteTableIdentifier.from(parentTablePath, parentDatabaseName, parentTableName,
- parentTableId);
+ return AbsoluteTableIdentifier.from(relation.getTablePath(), parentDatabaseName,
+ parentTableName, parentTableId);
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9ca985f0/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
index 8a0404c..04ec75d 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
@@ -17,6 +17,7 @@
package org.apache.spark.sql
+import scala.collection.JavaConverters._
import scala.collection.mutable.{ArrayBuffer, ListBuffer}
import scala.util.control.Breaks._
@@ -35,6 +36,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.indexstore.PartitionSpec
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
import org.apache.carbondata.core.scan.expression.Expression
import org.apache.carbondata.core.scan.expression.logical.AndExpression
import org.apache.carbondata.hadoop.CarbonProjection
@@ -80,6 +82,16 @@ case class CarbonDatasourceHadoopRelation(
val projection = new CarbonProjection
+ if (carbonTable.isChildDataMap) {
+ val parentTableIdentifier = carbonTable.getTableInfo.getParentRelationIdentifiers.get(0)
+ val path = CarbonEnv.getCarbonTable(Some(parentTableIdentifier.getDatabaseName),
+ parentTableIdentifier.getTableName)(sparkSession).getTablePath
+ for (carbonDimension: CarbonDimension <- carbonTable.getAllDimensions.asScala) {
+ carbonDimension.getColumnSchema.getParentColumnTableRelations.get(0)
+ .getRelationIdentifier.setTablePath(path)
+ }
+ }
+
// As Filter pushdown for Complex datatype is not supported, if filter is applied on complex
// column, then Projection pushdown on Complex Columns will not take effect. Hence, check if
// filter contains Struct Complex Column.