You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by zh...@apache.org on 2019/12/01 08:28:52 UTC
[carbondata] branch master updated: fix start/end key of no-dict
sort columns
This is an automated email from the ASF dual-hosted git repository.
zhangzc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new bacda1a fix start/end key of no-dict sort columns
bacda1a is described below
commit bacda1a6b57b3e515919584eeb85b290c021ec0a
Author: QiangCai <qi...@qq.com>
AuthorDate: Sat Nov 30 22:57:39 2019 +0800
fix start/end key of no-dict sort columns
This closes #3491
---
.../core/datastore/page/key/TablePageKey.java | 49 +++++++++++++++-------
.../core/metadata/schema/table/CarbonTable.java | 12 ++++++
2 files changed, 45 insertions(+), 16 deletions(-)
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/key/TablePageKey.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/key/TablePageKey.java
index 609f17e..3786ef1 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/key/TablePageKey.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/key/TablePageKey.java
@@ -18,18 +18,20 @@
package org.apache.carbondata.core.datastore.page.key;
import java.nio.ByteBuffer;
+import java.util.List;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.row.CarbonRow;
import org.apache.carbondata.core.datastore.row.WriteStepRowUtil;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
+import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.carbondata.core.util.NonDictionaryUtil;
public class TablePageKey {
private int pageSize;
- private byte[][] currentNoDictionaryKey;
-
// MDK start key
private byte[] startKey;
@@ -37,10 +39,10 @@ public class TablePageKey {
private byte[] endKey;
// startkey for no dictionary columns
- private byte[][] noDictStartKey;
+ private Object[] noDictStartKey;
// endkey for no diciotn
- private byte[][] noDictEndKey;
+ private Object[] noDictEndKey;
// startkey for no dictionary columns after packing into one column
private byte[] packedNoDictStartKey;
@@ -60,17 +62,17 @@ public class TablePageKey {
/** update all keys based on the input row */
public void update(int rowId, CarbonRow row, byte[] mdk) {
- if (hasNoDictionary) {
- Object[] noDictAndComplexDimension = WriteStepRowUtil.getNoDictAndComplexDimension(row);
- currentNoDictionaryKey = new byte[noDictAndComplexDimension.length][0];
- }
if (rowId == 0) {
startKey = mdk;
- noDictStartKey = currentNoDictionaryKey;
+ if (hasNoDictionary) {
+ noDictStartKey = WriteStepRowUtil.getNoDictAndComplexDimension(row);
+ }
}
- noDictEndKey = currentNoDictionaryKey;
if (rowId == pageSize - 1) {
endKey = mdk;
+ if (hasNoDictionary) {
+ noDictEndKey = WriteStepRowUtil.getNoDictAndComplexDimension(row);
+ }
finalizeKeys();
}
}
@@ -107,8 +109,8 @@ public class TablePageKey {
if (numberOfNoDictSortColumns > 0) {
// if sort_columns contain no-dictionary columns
if (noDictStartKey.length > numberOfNoDictSortColumns) {
- byte[][] newNoDictionaryStartKey = new byte[numberOfNoDictSortColumns][];
- byte[][] newNoDictionaryEndKey = new byte[numberOfNoDictSortColumns][];
+ Object[] newNoDictionaryStartKey = new Object[numberOfNoDictSortColumns];
+ Object[] newNoDictionaryEndKey = new Object[numberOfNoDictSortColumns];
System.arraycopy(
noDictStartKey, 0, newNoDictionaryStartKey, 0, numberOfNoDictSortColumns);
System.arraycopy(
@@ -116,10 +118,12 @@ public class TablePageKey {
noDictStartKey = newNoDictionaryStartKey;
noDictEndKey = newNoDictionaryEndKey;
}
- packedNoDictStartKey =
- NonDictionaryUtil.packByteBufferIntoSingleByteArray(noDictStartKey);
- packedNoDictEndKey =
- NonDictionaryUtil.packByteBufferIntoSingleByteArray(noDictEndKey);
+ List<CarbonDimension> noDictSortColumns =
+ CarbonTable.getNoDictSortColumns(segmentProperties.getDimensions());
+ packedNoDictStartKey = NonDictionaryUtil.packByteBufferIntoSingleByteArray(
+ convertKeys(noDictStartKey, noDictSortColumns));
+ packedNoDictEndKey = NonDictionaryUtil.packByteBufferIntoSingleByteArray(
+ convertKeys(noDictEndKey, noDictSortColumns));
} else {
noDictStartKey = new byte[0][];
noDictEndKey = new byte[0][];
@@ -128,6 +132,19 @@ public class TablePageKey {
}
}
+ private byte[][] convertKeys(Object[] keys, List<CarbonDimension> noDictSortColumns) {
+ byte[][] finalKeys = new byte[keys.length][];
+ for (int i = 0; i < keys.length; i++) {
+ if (keys[i] instanceof byte[]) {
+ finalKeys[i] = (byte[]) keys[i];
+ } else {
+ finalKeys[i] = DataTypeUtil.getBytesDataDataTypeForNoDictionaryColumn(keys[i],
+ noDictSortColumns.get(i).getDataType());
+ }
+ }
+ return finalKeys;
+ }
+
public byte[] getNoDictStartKey() {
return packedNoDictStartKey;
}
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index bb0b28d..9b9b756 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -818,6 +818,18 @@ public class CarbonTable implements Serializable, Writable {
return numberOfNoDictSortColumns;
}
+ public static List<CarbonDimension> getNoDictSortColumns(List<CarbonDimension> dimensions) {
+ List<CarbonDimension> noDictSortColumns = new ArrayList<>(dimensions.size());
+ for (int i = 0; i < dimensions.size(); i++) {
+ CarbonDimension dimension = dimensions.get(i);
+ if (dimension.isSortColumn() &&
+ !dimension.getColumnSchema().hasEncoding(Encoding.DICTIONARY)) {
+ noDictSortColumns.add(dimension);
+ }
+ }
+ return noDictSortColumns;
+ }
+
public CarbonColumn getRangeColumn() {
String rangeColumn =
tableInfo.getFactTable().getTableProperties().get(CarbonCommonConstants.RANGE_COLUMN);