You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ku...@apache.org on 2018/09/12 13:47:16 UTC
carbondata git commit: [HOTFIX] Old stores cannot read with new table
infered through sdk.
Repository: carbondata
Updated Branches:
refs/heads/master daa91c88e -> 4c692d185
[HOTFIX] Old stores cannot read with new table infered through sdk.
Problem: Old stores column schema is written in the different case then fileformat cannot read data because of sdk infer schema always gives lower case schema.
Solution: Do case insensitivity check while comparing.
It also disables prefetch as it is redundant for fileformat read and not getting inputmetrics properly if we use thread
This closes #2704
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/4c692d18
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/4c692d18
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/4c692d18
Branch: refs/heads/master
Commit: 4c692d185c4247e645d94c2d79787744c413817b
Parents: daa91c8
Author: ravipesala <ra...@gmail.com>
Authored: Mon Sep 10 21:11:18 2018 +0530
Committer: kumarvishal09 <ku...@gmail.com>
Committed: Wed Sep 12 19:16:39 2018 +0530
----------------------------------------------------------------------
.../apache/carbondata/core/metadata/CarbonMetadata.java | 5 +++--
.../metadata/schema/table/AggregationDataMapSchema.java | 4 ++--
.../core/metadata/schema/table/column/ColumnSchema.java | 2 +-
.../core/scan/executor/impl/AbstractQueryExecutor.java | 6 +++++-
.../core/scan/executor/util/RestructureUtil.java | 7 ++++---
.../scan/expression/logical/BinaryLogicalExpression.java | 2 +-
.../apache/carbondata/core/scan/filter/FilterUtil.java | 2 +-
.../org/apache/carbondata/core/scan/model/QueryModel.java | 10 ++++++++++
.../apache/carbondata/core/util/BlockletDataMapUtil.java | 2 +-
.../java/org/apache/carbondata/core/util/CarbonUtil.java | 2 +-
.../execution/datasources/SparkCarbonFileFormat.scala | 1 +
11 files changed, 30 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4c692d18/core/src/main/java/org/apache/carbondata/core/metadata/CarbonMetadata.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/CarbonMetadata.java b/core/src/main/java/org/apache/carbondata/core/metadata/CarbonMetadata.java
index 3f8c12d..850f477 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/CarbonMetadata.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/CarbonMetadata.java
@@ -143,7 +143,7 @@ public final class CarbonMetadata {
List<CarbonDimension> listOfCarbonDims =
carbonTable.getDimensionByTableName(carbonTable.getTableName());
for (CarbonDimension dimension : listOfCarbonDims) {
- if (dimension.getColumnId().equals(columnIdentifier)) {
+ if (dimension.getColumnId().equalsIgnoreCase(columnIdentifier)) {
return dimension;
}
if (dimension.getNumberOfChild() > 0) {
@@ -168,7 +168,8 @@ public final class CarbonMetadata {
private CarbonDimension getCarbonChildDimsBasedOnColIdentifier(String columnIdentifier,
CarbonDimension dimension) {
for (int i = 0; i < dimension.getNumberOfChild(); i++) {
- if (dimension.getListOfChildDimensions().get(i).getColumnId().equals(columnIdentifier)) {
+ if (dimension.getListOfChildDimensions().get(i).getColumnId()
+ .equalsIgnoreCase(columnIdentifier)) {
return dimension.getListOfChildDimensions().get(i);
} else if (dimension.getListOfChildDimensions().get(i).getNumberOfChild() > 0) {
CarbonDimension childDim = getCarbonChildDimsBasedOnColIdentifier(columnIdentifier,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4c692d18/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/AggregationDataMapSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/AggregationDataMapSchema.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/AggregationDataMapSchema.java
index 2bb6d18..c8bb5ad 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/AggregationDataMapSchema.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/AggregationDataMapSchema.java
@@ -152,7 +152,7 @@ public class AggregationDataMapSchema extends DataMapSchema {
List<ParentColumnTableRelation> parentColumnTableRelations =
columnSchema.getParentColumnTableRelations();
if (null != parentColumnTableRelations && parentColumnTableRelations.size() == 1
- && parentColumnTableRelations.get(0).getColumnName().equals(columName) &&
+ && parentColumnTableRelations.get(0).getColumnName().equalsIgnoreCase(columName) &&
columnSchema.getColumnName().endsWith(columName)) {
return columnSchema;
}
@@ -198,7 +198,7 @@ public class AggregationDataMapSchema extends DataMapSchema {
List<ParentColumnTableRelation> parentColumnTableRelations =
columnSchema.getParentColumnTableRelations();
if (null != parentColumnTableRelations && parentColumnTableRelations.size() == 1
- && parentColumnTableRelations.get(0).getColumnName().equals(columName)
+ && parentColumnTableRelations.get(0).getColumnName().equalsIgnoreCase(columName)
&& timeseriesFunction.equalsIgnoreCase(columnSchema.getTimeSeriesFunction())) {
return columnSchema;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4c692d18/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
index 853851d..8eba101 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
@@ -315,7 +315,7 @@ public class ColumnSchema implements Serializable, Writable {
if (other.columnName != null) {
return false;
}
- } else if (!columnName.equals(other.columnName)) {
+ } else if (!columnName.equalsIgnoreCase(other.columnName)) {
return false;
}
if (dataType == null) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4c692d18/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
index bd1eb1c..83276e0 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
@@ -476,7 +476,11 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
blockExecutionInfo
.setTotalNumberDimensionToRead(
segmentProperties.getDimensionOrdinalToChunkMapping().size());
- blockExecutionInfo.setPrefetchBlocklet(!queryModel.isReadPageByPage());
+ if (queryModel.isReadPageByPage()) {
+ blockExecutionInfo.setPrefetchBlocklet(false);
+ } else {
+ blockExecutionInfo.setPrefetchBlocklet(queryModel.isPreFetchData());
+ }
blockExecutionInfo
.setTotalNumberOfMeasureToRead(segmentProperties.getMeasuresOrdinalToChunkMapping().size());
blockExecutionInfo.setComplexDimensionInfoMap(QueryUtil
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4c692d18/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
index 9fe2a38..e823eb2 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
@@ -165,14 +165,15 @@ public class RestructureUtil {
// column ID but can have same column name
if (tableColumn.getDataType().isComplexType() && !(tableColumn.getDataType().getId()
== DataTypes.ARRAY_TYPE_ID)) {
- if (tableColumn.getColumnId().equals(queryColumn.getColumnId())) {
+ if (tableColumn.getColumnId().equalsIgnoreCase(queryColumn.getColumnId())) {
return true;
} else {
return isColumnMatchesStruct(tableColumn, queryColumn);
}
} else {
- return (tableColumn.getColumnId().equals(queryColumn.getColumnId()) || (!isTransactionalTable
- && tableColumn.getColName().equals(queryColumn.getColName())));
+ return (tableColumn.getColumnId().equalsIgnoreCase(queryColumn.getColumnId()) || (
+ !isTransactionalTable && tableColumn.getColName()
+ .equalsIgnoreCase(queryColumn.getColName())));
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4c692d18/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/BinaryLogicalExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/BinaryLogicalExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/BinaryLogicalExpression.java
index 1a5b6a7..ecbb8d9 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/BinaryLogicalExpression.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/BinaryLogicalExpression.java
@@ -64,7 +64,7 @@ public abstract class BinaryLogicalExpression extends BinaryExpression {
boolean found = false;
for (ColumnExpression currentColExp : lst) {
- if (currentColExp.getColumnName().equals(colExp.getColumnName())) {
+ if (currentColExp.getColumnName().equalsIgnoreCase(colExp.getColumnName())) {
found = true;
colExp.setColIndex(currentColExp.getColIndex());
break;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4c692d18/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
index bae608f..410f67a 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
@@ -397,7 +397,7 @@ public final class FilterUtil {
int columnIndexInMinMaxByteArray = -1;
int columnCounter = 0;
for (CarbonColumn cachedColumn : carbonDimensionsToBeCached) {
- if (cachedColumn.getColumnId().equals(filterColumn.getColumnId())) {
+ if (cachedColumn.getColumnId().equalsIgnoreCase(filterColumn.getColumnId())) {
columnIndexInMinMaxByteArray = columnCounter;
break;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4c692d18/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
index 6df98e6..d90c35e 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
@@ -122,6 +122,8 @@ public class QueryModel {
// whether to clear/free unsafe memory or not
private boolean freeUnsafeMemory = true;
+ private boolean preFetchData = true;
+
private QueryModel(CarbonTable carbonTable) {
tableBlockInfos = new ArrayList<TableBlockInfo>();
invalidSegmentIds = new ArrayList<>();
@@ -396,6 +398,14 @@ public class QueryModel {
isFG = FG;
}
+ public boolean isPreFetchData() {
+ return preFetchData;
+ }
+
+ public void setPreFetchData(boolean preFetchData) {
+ this.preFetchData = preFetchData;
+ }
+
@Override
public String toString() {
return String.format("scan on table %s.%s, %d projection columns with filter (%s)",
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4c692d18/core/src/main/java/org/apache/carbondata/core/util/BlockletDataMapUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/BlockletDataMapUtil.java b/core/src/main/java/org/apache/carbondata/core/util/BlockletDataMapUtil.java
index 1832cf5..4a87e91 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/BlockletDataMapUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/BlockletDataMapUtil.java
@@ -479,7 +479,7 @@ public class BlockletDataMapUtil {
private static boolean filterColumnExistsInMinMaxColumnList(List<CarbonColumn> minMaxCacheColumns,
CarbonColumn filterColumn) {
for (CarbonColumn column : minMaxCacheColumns) {
- if (filterColumn.getColumnId().equals(column.getColumnId())) {
+ if (filterColumn.getColumnId().equalsIgnoreCase(column.getColumnId())) {
return true;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4c692d18/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index d71f294..b1e3ef0 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -1193,7 +1193,7 @@ public final class CarbonUtil {
List<CarbonDimension> blockDimensions, CarbonDimension dimensionToBeSearched) {
CarbonDimension currentBlockDimension = null;
for (CarbonDimension blockDimension : blockDimensions) {
- if (dimensionToBeSearched.getColumnId().equals(blockDimension.getColumnId())) {
+ if (dimensionToBeSearched.getColumnId().equalsIgnoreCase(blockDimension.getColumnId())) {
currentBlockDimension = blockDimension;
break;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4c692d18/integration/spark-datasource/src/main/scala/org/apache/spark/sql/carbondata/execution/datasources/SparkCarbonFileFormat.scala
----------------------------------------------------------------------
diff --git a/integration/spark-datasource/src/main/scala/org/apache/spark/sql/carbondata/execution/datasources/SparkCarbonFileFormat.scala b/integration/spark-datasource/src/main/scala/org/apache/spark/sql/carbondata/execution/datasources/SparkCarbonFileFormat.scala
index 62d9903..a6965ac 100644
--- a/integration/spark-datasource/src/main/scala/org/apache/spark/sql/carbondata/execution/datasources/SparkCarbonFileFormat.scala
+++ b/integration/spark-datasource/src/main/scala/org/apache/spark/sql/carbondata/execution/datasources/SparkCarbonFileFormat.scala
@@ -387,6 +387,7 @@ class SparkCarbonFileFormat extends FileFormat
new TaskAttemptContextImpl(broadcastedHadoopConf.value.value, attemptId)
val model = format.createQueryModel(split, hadoopAttemptContext)
model.setConverter(new SparkDataTypeConverterImpl)
+ model.setPreFetchData(false)
val carbonReader = if (readVector) {
val vectorizedReader = new VectorizedCarbonRecordReader(model,
null,