You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2018/06/22 07:17:20 UTC
carbondata git commit: [CARBONDATA-2626] Fix bugs in sequence of
column page
Repository: carbondata
Updated Branches:
refs/heads/master b3f782062 -> 18dc3ff49
[CARBONDATA-2626] Fix bugs in sequence of column page
In tablespec, the first dimension spec is 'name' which is sort_columns but not dictionary. But Carbondata now treat it as dictionary page. (See TablePage Line94). This PR fixes this bug
This closes #2393
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/18dc3ff4
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/18dc3ff4
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/18dc3ff4
Branch: refs/heads/master
Commit: 18dc3ff49b23d071a545e4645f4565a45274c587
Parents: b3f7820
Author: xuchuanyin <xu...@hust.edu.cn>
Authored: Thu Jun 21 17:51:30 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Fri Jun 22 15:17:09 2018 +0800
----------------------------------------------------------------------
.../carbondata/processing/store/TablePage.java | 38 +++++++++++---------
1 file changed, 22 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/18dc3ff4/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java b/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java
index 1a1c5d1..b1b966b 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java
@@ -89,28 +89,34 @@ public class TablePage {
this.pageSize = pageSize;
int numDictDimension = model.getMDKeyGenerator().getDimCount();
TableSpec tableSpec = model.getTableSpec();
+
dictDimensionPages = new ColumnPage[numDictDimension];
- for (int i = 0; i < dictDimensionPages.length; i++) {
- TableSpec.DimensionSpec spec = tableSpec.getDimensionSpec(i);
- ColumnPage page = ColumnPage.newPage(spec, DataTypes.BYTE_ARRAY, pageSize);
- page.setStatsCollector(KeyPageStatsCollector.newInstance(DataTypes.BYTE_ARRAY));
- dictDimensionPages[i] = page;
- }
noDictDimensionPages = new ColumnPage[model.getNoDictionaryCount()];
- for (int i = 0; i < noDictDimensionPages.length; i++) {
- TableSpec.DimensionSpec spec = tableSpec.getDimensionSpec(i + numDictDimension);
+ int tmpNumDictDimIdx = 0;
+ int tmpNumNoDictDimIdx = 0;
+ for (int i = 0; i < dictDimensionPages.length + noDictDimensionPages.length; i++) {
+ TableSpec.DimensionSpec spec = tableSpec.getDimensionSpec(i);
+ ColumnType columnType = tableSpec.getDimensionSpec(i).getColumnType();
ColumnPage page;
- if (DataTypes.VARCHAR == spec.getSchemaDataType()) {
- page = ColumnPage.newPage(spec, DataTypes.VARCHAR, pageSize);
- page.setStatsCollector(LVLongStringStatsCollector.newInstance());
+ if (ColumnType.GLOBAL_DICTIONARY == columnType
+ || ColumnType.DIRECT_DICTIONARY == columnType) {
+ page = ColumnPage.newPage(spec, DataTypes.BYTE_ARRAY, pageSize);
+ page.setStatsCollector(KeyPageStatsCollector.newInstance(DataTypes.BYTE_ARRAY));
+ dictDimensionPages[tmpNumDictDimIdx++] = page;
} else {
- // In previous implementation, other data types such as string, date and timestamp
- // will be encoded using string page
- page = ColumnPage.newPage(spec, DataTypes.STRING, pageSize);
- page.setStatsCollector(LVShortStringStatsCollector.newInstance());
+ if (DataTypes.VARCHAR == spec.getSchemaDataType()) {
+ page = ColumnPage.newPage(spec, DataTypes.VARCHAR, pageSize);
+ page.setStatsCollector(LVLongStringStatsCollector.newInstance());
+ } else {
+ // In previous implementation, other data types such as string, date and timestamp
+ // will be encoded using string page
+ page = ColumnPage.newPage(spec, DataTypes.STRING, pageSize);
+ page.setStatsCollector(LVShortStringStatsCollector.newInstance());
+ }
+ noDictDimensionPages[tmpNumNoDictDimIdx++] = page;
}
- noDictDimensionPages[i] = page;
}
+
complexDimensionPages = new ComplexColumnPage[model.getComplexColumnCount()];
for (int i = 0; i < complexDimensionPages.length; i++) {
// here we still do not the depth of the complex column, it will be initialized when