You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/06/20 07:29:47 UTC
[41/56] [abbrv] carbondata git commit: rename RLE
rename RLE
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/6a09ee84
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/6a09ee84
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/6a09ee84
Branch: refs/heads/streaming_ingest
Commit: 6a09ee84c6efde7675f1e49aaa1adf1a7ff1ba34
Parents: 7bb8220
Author: jackylk <ja...@huawei.com>
Authored: Tue Jun 13 21:56:06 2017 +0800
Committer: QiangCai <qi...@qq.com>
Committed: Sun Jun 18 01:27:02 2017 +0800
----------------------------------------------------------------------
.../columnar/BlockIndexerStorageForInt.java | 141 +++++++++----------
.../BlockIndexerStorageForNoInvertedIndex.java | 104 --------------
...kIndexerStorageForNoInvertedIndexForInt.java | 103 ++++++++++++++
...ndexerStorageForNoInvertedIndexForShort.java | 48 +++----
.../columnar/BlockIndexerStorageForShort.java | 135 +++++++++---------
.../datastore/columnar/ColumnWithIntIndex.java | 80 -----------
.../columnar/ColumnWithIntIndexForHighCard.java | 47 -------
.../datastore/columnar/ColumnWithRowId.java | 77 ++++++++++
.../columnar/ColumnWithRowIdForHighCard.java | 47 +++++++
.../columnar/ColumnWithShortIndex.java | 76 ----------
.../ColumnWithShortIndexForNoDictionay.java | 46 ------
.../core/datastore/columnar/IndexStorage.java | 8 +-
.../columnar/ColGroupBlockStorage.java | 25 ++--
.../processing/store/TablePageEncoder.java | 16 +--
.../writer/v1/CarbonFactDataWriterImplV1.java | 10 +-
.../writer/v3/CarbonFactDataWriterImplV3.java | 10 +-
16 files changed, 421 insertions(+), 552 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6a09ee84/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForInt.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForInt.java
index 826f6d2..78e6d35 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForInt.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForInt.java
@@ -27,61 +27,56 @@ import org.apache.carbondata.core.util.ByteUtil;
public class BlockIndexerStorageForInt implements IndexStorage<int[]> {
private boolean alreadySorted;
- private int[] dataAfterComp;
+ private int[] rowIdPage;
- private int[] indexMap;
+ private int[] rowIdRlePage;
- private byte[][] keyBlock;
+ private byte[][] dataPage;
- private int[] dataIndexMap;
+ private int[] dataRlePage;
private int totalSize;
- public BlockIndexerStorageForInt(byte[][] keyBlock, boolean compressData, boolean isNoDictionary,
+ public BlockIndexerStorageForInt(byte[][] dataPage, boolean rleOnData, boolean isNoDictionary,
boolean isSortRequired) {
- ColumnWithIntIndex[] columnWithIndexs = createColumnWithIndexArray(keyBlock, isNoDictionary);
+ ColumnWithRowId<Integer>[] dataWithRowId = createColumnWithRowId(dataPage, isNoDictionary);
if (isSortRequired) {
- Arrays.sort(columnWithIndexs);
+ Arrays.sort(dataWithRowId);
}
- compressMyOwnWay(extractDataAndReturnIndexes(columnWithIndexs, keyBlock));
- if (compressData) {
- compressDataMyOwnWay(columnWithIndexs);
+ int[] rowIds = extractDataAndReturnRowIds(dataWithRowId, dataPage);
+ rleEncodeOnRowId(rowIds);
+ if (rleOnData) {
+ rleEncodeOnData(dataWithRowId);
}
}
/**
- * Create an object with each column array and respective index
- *
- * @return
+ * Create an object with each column array and respective rowId
*/
- private ColumnWithIntIndex[] createColumnWithIndexArray(byte[][] keyBlock,
+ private ColumnWithRowId<Integer>[] createColumnWithRowId(byte[][] dataPage,
boolean isNoDictionary) {
- ColumnWithIntIndex[] columnWithIndexs;
+ ColumnWithRowId<Integer>[] columnWithRowId = new ColumnWithRowId[dataPage.length];
if (isNoDictionary) {
- columnWithIndexs = new ColumnWithIntIndexForHighCard[keyBlock.length];
- for (int i = 0; i < columnWithIndexs.length; i++) {
- columnWithIndexs[i] = new ColumnWithIntIndexForHighCard(keyBlock[i], i);
+ for (int i = 0; i < columnWithRowId.length; i++) {
+ columnWithRowId[i] = new ColumnWithRowIdForHighCard<>(dataPage[i], i);
}
-
} else {
- columnWithIndexs = new ColumnWithIntIndex[keyBlock.length];
- for (int i = 0; i < columnWithIndexs.length; i++) {
- columnWithIndexs[i] = new ColumnWithIntIndex(keyBlock[i], i);
+ for (int i = 0; i < columnWithRowId.length; i++) {
+ columnWithRowId[i] = new ColumnWithRowId<>(dataPage[i], i);
}
}
-
- return columnWithIndexs;
+ return columnWithRowId;
}
- private int[] extractDataAndReturnIndexes(ColumnWithIntIndex[] columnWithIndexs,
+ private int[] extractDataAndReturnRowIds(ColumnWithRowId<Integer>[] dataWithRowId,
byte[][] keyBlock) {
- int[] indexes = new int[columnWithIndexs.length];
- for (int i = 0; i < indexes.length; i++) {
- indexes[i] = columnWithIndexs[i].getIndex();
- keyBlock[i] = columnWithIndexs[i].getColumn();
+ int[] rowId = new int[dataWithRowId.length];
+ for (int i = 0; i < rowId.length; i++) {
+ rowId[i] = dataWithRowId[i].getIndex();
+ keyBlock[i] = dataWithRowId[i].getColumn();
}
- this.keyBlock = keyBlock;
- return indexes;
+ this.dataPage = keyBlock;
+ return rowId;
}
/**
@@ -92,41 +87,41 @@ public class BlockIndexerStorageForInt implements IndexStorage<int[]> {
* sequential numbers then the same array it returns with empty second
* array.
*
- * @param indexes
+ * @param rowIds
*/
- public void compressMyOwnWay(int[] indexes) {
+ public void rleEncodeOnRowId(int[] rowIds) {
List<Integer> list = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
List<Integer> map = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
int k = 0;
int i = 1;
- for (; i < indexes.length; i++) {
- if (indexes[i] - indexes[i - 1] == 1) {
+ for (; i < rowIds.length; i++) {
+ if (rowIds[i] - rowIds[i - 1] == 1) {
k++;
} else {
if (k > 0) {
map.add((list.size()));
- list.add(indexes[i - k - 1]);
- list.add(indexes[i - 1]);
+ list.add(rowIds[i - k - 1]);
+ list.add(rowIds[i - 1]);
} else {
- list.add(indexes[i - 1]);
+ list.add(rowIds[i - 1]);
}
k = 0;
}
}
if (k > 0) {
map.add((list.size()));
- list.add(indexes[i - k - 1]);
- list.add(indexes[i - 1]);
+ list.add(rowIds[i - k - 1]);
+ list.add(rowIds[i - 1]);
} else {
- list.add(indexes[i - 1]);
+ list.add(rowIds[i - 1]);
}
- dataAfterComp = convertToArray(list);
- if (indexes.length == dataAfterComp.length) {
- indexMap = new int[0];
+ rowIdPage = convertToArray(list);
+ if (rowIds.length == rowIdPage.length) {
+ rowIdRlePage = new int[0];
} else {
- indexMap = convertToArray(map);
+ rowIdRlePage = convertToArray(map);
}
- if (dataAfterComp.length == 2 && indexMap.length == 1) {
+ if (rowIdPage.length == 2 && rowIdRlePage.length == 1) {
alreadySorted = true;
}
}
@@ -147,38 +142,38 @@ public class BlockIndexerStorageForInt implements IndexStorage<int[]> {
}
/**
- * @return the dataAfterComp
+ * @return the rowIdPage
*/
- public int[] getDataAfterComp() {
- return dataAfterComp;
+ public int[] getRowIdPage() {
+ return rowIdPage;
}
/**
- * @return the indexMap
+ * @return the rowIdRlePage
*/
- public int[] getIndexMap() {
- return indexMap;
+ public int[] getRowIdRlePage() {
+ return rowIdRlePage;
}
/**
- * @return the keyBlock
+ * @return the dataPage
*/
- public byte[][] getKeyBlock() {
- return keyBlock;
+ public byte[][] getDataPage() {
+ return dataPage;
}
- private void compressDataMyOwnWay(ColumnWithIntIndex[] indexes) {
- byte[] prvKey = indexes[0].getColumn();
- List<ColumnWithIntIndex> list =
- new ArrayList<ColumnWithIntIndex>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
- list.add(indexes[0]);
+ private void rleEncodeOnData(ColumnWithRowId[] dataWithRowId) {
+ byte[] prvKey = dataWithRowId[0].getColumn();
+ List<ColumnWithRowId> list =
+ new ArrayList<ColumnWithRowId>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+ list.add(dataWithRowId[0]);
int counter = 1;
int start = 0;
List<Integer> map = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
- for (int i = 1; i < indexes.length; i++) {
- if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(prvKey, indexes[i].getColumn()) != 0) {
- prvKey = indexes[i].getColumn();
- list.add(indexes[i]);
+ for (int i = 1; i < dataWithRowId.length; i++) {
+ if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(prvKey, dataWithRowId[i].getColumn()) != 0) {
+ prvKey = dataWithRowId[i].getColumn();
+ list.add(dataWithRowId[i]);
map.add(start);
map.add(counter);
start += counter;
@@ -189,15 +184,15 @@ public class BlockIndexerStorageForInt implements IndexStorage<int[]> {
}
map.add(start);
map.add(counter);
- this.keyBlock = convertToKeyArray(list);
- if (indexes.length == keyBlock.length) {
- dataIndexMap = new int[0];
+ this.dataPage = convertToDataPage(list);
+ if (dataWithRowId.length == dataPage.length) {
+ dataRlePage = new int[0];
} else {
- dataIndexMap = convertToArray(map);
+ dataRlePage = convertToArray(map);
}
}
- private byte[][] convertToKeyArray(List<ColumnWithIntIndex> list) {
+ private byte[][] convertToDataPage(List<ColumnWithRowId> list) {
byte[][] shortArray = new byte[list.size()][];
for (int i = 0; i < shortArray.length; i++) {
shortArray[i] = list.get(i).getColumn();
@@ -206,8 +201,8 @@ public class BlockIndexerStorageForInt implements IndexStorage<int[]> {
return shortArray;
}
- @Override public int[] getDataIndexMap() {
- return dataIndexMap;
+ public int[] getDataRlePage() {
+ return dataRlePage;
}
@Override public int getTotalSize() {
@@ -215,10 +210,10 @@ public class BlockIndexerStorageForInt implements IndexStorage<int[]> {
}
@Override public byte[] getMin() {
- return keyBlock[0];
+ return dataPage[0];
}
@Override public byte[] getMax() {
- return keyBlock[keyBlock.length - 1];
+ return dataPage[dataPage.length - 1];
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6a09ee84/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java
deleted file mode 100644
index f36ee97..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.datastore.columnar;
-
-import org.apache.carbondata.core.datastore.columnar.IndexStorage;
-import org.apache.carbondata.core.util.ByteUtil;
-
-/**
- * Below class will be used to for no inverted index
- */
-public class BlockIndexerStorageForNoInvertedIndex implements IndexStorage<int[]> {
-
- /**
- * column data
- */
- private byte[][] keyBlock;
-
- /**
- * total number of rows
- */
- private int totalSize;
-
- private byte[] min;
- private byte[] max;
-
- public BlockIndexerStorageForNoInvertedIndex(byte[][] keyBlockInput) {
- this.keyBlock = keyBlockInput;
- min = keyBlock[0];
- max = keyBlock[0];
- totalSize += keyBlock[0].length;
- int minCompare = 0;
- int maxCompare = 0;
- for (int i = 1; i < keyBlock.length; i++) {
- totalSize += keyBlock[i].length;
- minCompare = ByteUtil.compare(min, keyBlock[i]);
- maxCompare = ByteUtil.compare(max, keyBlock[i]);
- if (minCompare > 0) {
- min = keyBlock[i];
- }
- if (maxCompare < 0) {
- max = keyBlock[i];
- }
- }
- }
-
- @Override public int[] getDataIndexMap() {
- return new int[0];
- }
-
- @Override public int getTotalSize() {
- return totalSize;
- }
-
- @Override public boolean isAlreadySorted() {
- return true;
- }
-
- /**
- * no use
- *
- * @return
- */
- @Override public int[] getDataAfterComp() {
- return new int[0];
- }
-
- /**
- * no use
- *
- * @return
- */
- @Override public int[] getIndexMap() {
- return new int[0];
- }
-
- /**
- * @return the keyBlock
- */
- public byte[][] getKeyBlock() {
- return keyBlock;
- }
-
- @Override public byte[] getMin() {
- return min;
- }
-
- @Override public byte[] getMax() {
- return max;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6a09ee84/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndexForInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndexForInt.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndexForInt.java
new file mode 100644
index 0000000..c168614
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndexForInt.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datastore.columnar;
+
+import org.apache.carbondata.core.util.ByteUtil;
+
+/**
+ * Below class will be used to for no inverted index
+ */
+public class BlockIndexerStorageForNoInvertedIndexForInt implements IndexStorage<int[]> {
+
+ /**
+ * column data
+ */
+ private byte[][] dataPage;
+
+ /**
+ * total number of rows
+ */
+ private int totalSize;
+
+ private byte[] min;
+ private byte[] max;
+
+ public BlockIndexerStorageForNoInvertedIndexForInt(byte[][] dataPage) {
+ this.dataPage = dataPage;
+ min = this.dataPage[0];
+ max = this.dataPage[0];
+ totalSize += this.dataPage[0].length;
+ int minCompare = 0;
+ int maxCompare = 0;
+ for (int i = 1; i < this.dataPage.length; i++) {
+ totalSize += this.dataPage[i].length;
+ minCompare = ByteUtil.compare(min, this.dataPage[i]);
+ maxCompare = ByteUtil.compare(max, this.dataPage[i]);
+ if (minCompare > 0) {
+ min = this.dataPage[i];
+ }
+ if (maxCompare < 0) {
+ max = this.dataPage[i];
+ }
+ }
+ }
+
+ public int[] getDataRlePage() {
+ return new int[0];
+ }
+
+ @Override public int getTotalSize() {
+ return totalSize;
+ }
+
+ @Override public boolean isAlreadySorted() {
+ return true;
+ }
+
+ /**
+ * no use
+ *
+ * @return
+ */
+ public int[] getRowIdPage() {
+ return new int[0];
+ }
+
+ /**
+ * no use
+ *
+ * @return
+ */
+ public int[] getRowIdRlePage() {
+ return new int[0];
+ }
+
+ /**
+ * @return the dataPage
+ */
+ public byte[][] getDataPage() {
+ return dataPage;
+ }
+
+ @Override public byte[] getMin() {
+ return min;
+ }
+
+ @Override public byte[] getMax() {
+ return max;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6a09ee84/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndexForShort.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndexForShort.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndexForShort.java
index f14338b..5712655 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndexForShort.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndexForShort.java
@@ -26,7 +26,7 @@ public class BlockIndexerStorageForNoInvertedIndexForShort implements IndexStora
/**
* column data
*/
- private byte[][] keyBlock;
+ private byte[][] dataPage;
/**
* total number of rows
@@ -36,44 +36,44 @@ public class BlockIndexerStorageForNoInvertedIndexForShort implements IndexStora
private byte[] min;
private byte[] max;
- public BlockIndexerStorageForNoInvertedIndexForShort(byte[][] keyBlockInput,
+ public BlockIndexerStorageForNoInvertedIndexForShort(byte[][] dataPage,
boolean isNoDictonary) {
- this.keyBlock = keyBlockInput;
- min = keyBlock[0];
- max = keyBlock[0];
- totalSize += keyBlock[0].length;
+ this.dataPage = dataPage;
+ min = this.dataPage[0];
+ max = this.dataPage[0];
+ totalSize += this.dataPage[0].length;
int minCompare = 0;
int maxCompare = 0;
if (!isNoDictonary) {
- for (int i = 1; i < keyBlock.length; i++) {
- totalSize += keyBlock[i].length;
- minCompare = ByteUtil.compare(min, keyBlock[i]);
- maxCompare = ByteUtil.compare(max, keyBlock[i]);
+ for (int i = 1; i < this.dataPage.length; i++) {
+ totalSize += this.dataPage[i].length;
+ minCompare = ByteUtil.compare(min, this.dataPage[i]);
+ maxCompare = ByteUtil.compare(max, this.dataPage[i]);
if (minCompare > 0) {
- min = keyBlock[i];
+ min = this.dataPage[i];
}
if (maxCompare < 0) {
- max = keyBlock[i];
+ max = this.dataPage[i];
}
}
} else {
- for (int i = 1; i < keyBlock.length; i++) {
- totalSize += keyBlock[i].length;
+ for (int i = 1; i < this.dataPage.length; i++) {
+ totalSize += this.dataPage[i].length;
minCompare = ByteUtil.UnsafeComparer.INSTANCE
- .compareTo(min, 2, min.length - 2, keyBlock[i], 2, keyBlock[i].length - 2);
+ .compareTo(min, 2, min.length - 2, this.dataPage[i], 2, this.dataPage[i].length - 2);
maxCompare = ByteUtil.UnsafeComparer.INSTANCE
- .compareTo(max, 2, max.length - 2, keyBlock[i], 2, keyBlock[i].length - 2);
+ .compareTo(max, 2, max.length - 2, this.dataPage[i], 2, this.dataPage[i].length - 2);
if (minCompare > 0) {
- min = keyBlock[i];
+ min = this.dataPage[i];
}
if (maxCompare < 0) {
- max = keyBlock[i];
+ max = this.dataPage[i];
}
}
}
}
- @Override public short[] getDataIndexMap() {
+ public short[] getDataRlePage() {
return new short[0];
}
@@ -90,7 +90,7 @@ public class BlockIndexerStorageForNoInvertedIndexForShort implements IndexStora
*
* @return
*/
- @Override public short[] getDataAfterComp() {
+ public short[] getRowIdPage() {
return new short[0];
}
@@ -99,15 +99,15 @@ public class BlockIndexerStorageForNoInvertedIndexForShort implements IndexStora
*
* @return
*/
- @Override public short[] getIndexMap() {
+ public short[] getRowIdRlePage() {
return new short[0];
}
/**
- * @return the keyBlock
+ * @return the dataPage
*/
- public byte[][] getKeyBlock() {
- return keyBlock;
+ public byte[][] getDataPage() {
+ return dataPage;
}
@Override public byte[] getMin() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6a09ee84/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForShort.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForShort.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForShort.java
index aaea62b..2ffb610 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForShort.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForShort.java
@@ -27,58 +27,57 @@ public class BlockIndexerStorageForShort implements IndexStorage<short[]> {
private boolean alreadySorted;
- private short[] dataAfterComp;
+ private short[] rowIdPage;
- private short[] indexMap;
+ private short[] rowIdRlePage;
- private byte[][] keyBlock;
+ private byte[][] dataPage;
- private short[] dataIndexMap;
+ private short[] dataRlePage;
private int totalSize;
- public BlockIndexerStorageForShort(byte[][] keyBlock, boolean compressData,
+ public BlockIndexerStorageForShort(byte[][] dataPage, boolean rleOnData,
boolean isNoDictionary, boolean isSortRequired) {
- ColumnWithShortIndex[] columnWithIndexs = createColumnWithIndexArray(keyBlock, isNoDictionary);
+ ColumnWithRowId<Short>[] dataWithRowId = createColumnWithRowId(dataPage, isNoDictionary);
if (isSortRequired) {
- Arrays.sort(columnWithIndexs);
+ Arrays.sort(dataWithRowId);
}
- compressMyOwnWay(extractDataAndReturnIndexes(columnWithIndexs, keyBlock));
- if (compressData) {
- compressDataMyOwnWay(columnWithIndexs);
+ short[] rowIds = extractDataAndReturnRowId(dataWithRowId, dataPage);
+ rleEncodeOnRowId(rowIds);
+ if (rleOnData) {
+ rleEncodeOnData(dataWithRowId);
}
}
/**
- * Create an object with each column array and respective index
+ * Create an object with each column array and respective rowId
*
* @return
*/
- private ColumnWithShortIndex[] createColumnWithIndexArray(byte[][] keyBlock,
+ private ColumnWithRowId<Short>[] createColumnWithRowId(byte[][] dataPage,
boolean isNoDictionary) {
- ColumnWithShortIndex[] columnWithIndexs;
+ ColumnWithRowId<Short>[] columnWithIndexs = new ColumnWithRowId[dataPage.length];
if (isNoDictionary) {
- columnWithIndexs = new ColumnWithShortIndex[keyBlock.length];
for (short i = 0; i < columnWithIndexs.length; i++) {
- columnWithIndexs[i] = new ColumnWithShortIndexForNoDictionay(keyBlock[i], i);
+ columnWithIndexs[i] = new ColumnWithRowIdForHighCard<>(dataPage[i], i);
}
} else {
- columnWithIndexs = new ColumnWithShortIndex[keyBlock.length];
for (short i = 0; i < columnWithIndexs.length; i++) {
- columnWithIndexs[i] = new ColumnWithShortIndex(keyBlock[i], i);
+ columnWithIndexs[i] = new ColumnWithRowId<>(dataPage[i], i);
}
}
return columnWithIndexs;
}
- private short[] extractDataAndReturnIndexes(ColumnWithShortIndex[] columnWithIndexs,
- byte[][] keyBlock) {
- short[] indexes = new short[columnWithIndexs.length];
+ private short[] extractDataAndReturnRowId(ColumnWithRowId<Short>[] dataWithRowId,
+ byte[][] dataPage) {
+ short[] indexes = new short[dataWithRowId.length];
for (int i = 0; i < indexes.length; i++) {
- indexes[i] = columnWithIndexs[i].getIndex();
- keyBlock[i] = columnWithIndexs[i].getColumn();
+ indexes[i] = dataWithRowId[i].getIndex();
+ dataPage[i] = dataWithRowId[i].getColumn();
}
- this.keyBlock = keyBlock;
+ this.dataPage = dataPage;
return indexes;
}
@@ -90,46 +89,46 @@ public class BlockIndexerStorageForShort implements IndexStorage<short[]> {
* sequential numbers then the same array it returns with empty second
* array.
*
- * @param indexes
+ * @param rowIds
*/
- private void compressMyOwnWay(short[] indexes) {
+ private void rleEncodeOnRowId(short[] rowIds) {
List<Short> list = new ArrayList<Short>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
List<Short> map = new ArrayList<Short>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
int k = 0;
int i = 1;
- for (; i < indexes.length; i++) {
- if (indexes[i] - indexes[i - 1] == 1) {
+ for (; i < rowIds.length; i++) {
+ if (rowIds[i] - rowIds[i - 1] == 1) {
k++;
} else {
if (k > 0) {
map.add(((short) list.size()));
- list.add(indexes[i - k - 1]);
- list.add(indexes[i - 1]);
+ list.add(rowIds[i - k - 1]);
+ list.add(rowIds[i - 1]);
} else {
- list.add(indexes[i - 1]);
+ list.add(rowIds[i - 1]);
}
k = 0;
}
}
if (k > 0) {
map.add(((short) list.size()));
- list.add(indexes[i - k - 1]);
- list.add(indexes[i - 1]);
+ list.add(rowIds[i - k - 1]);
+ list.add(rowIds[i - 1]);
} else {
- list.add(indexes[i - 1]);
+ list.add(rowIds[i - 1]);
}
- double compressionPercentage = (((list.size() + map.size()) * 100) / indexes.length);
+ double compressionPercentage = (((list.size() + map.size()) * 100) / rowIds.length);
if (compressionPercentage > 70) {
- dataAfterComp = indexes;
+ rowIdPage = rowIds;
} else {
- dataAfterComp = convertToArray(list);
+ rowIdPage = convertToArray(list);
}
- if (indexes.length == dataAfterComp.length) {
- indexMap = new short[0];
+ if (rowIds.length == rowIdPage.length) {
+ rowIdRlePage = new short[0];
} else {
- indexMap = convertToArray(map);
+ rowIdRlePage = convertToArray(map);
}
- if (dataAfterComp.length == 2 && indexMap.length == 1) {
+ if (rowIdPage.length == 2 && rowIdRlePage.length == 1) {
alreadySorted = true;
}
}
@@ -150,37 +149,37 @@ public class BlockIndexerStorageForShort implements IndexStorage<short[]> {
}
/**
- * @return the dataAfterComp
+ * @return the rowIdPage
*/
- public short[] getDataAfterComp() {
- return dataAfterComp;
+ public short[] getRowIdPage() {
+ return rowIdPage;
}
/**
- * @return the indexMap
+ * @return the rowIdRlePage
*/
- public short[] getIndexMap() {
- return indexMap;
+ public short[] getRowIdRlePage() {
+ return rowIdRlePage;
}
/**
- * @return the keyBlock
+ * @return the dataPage
*/
- public byte[][] getKeyBlock() {
- return keyBlock;
+ public byte[][] getDataPage() {
+ return dataPage;
}
- private void compressDataMyOwnWay(ColumnWithShortIndex[] indexes) {
- byte[] prvKey = indexes[0].getColumn();
- List<ColumnWithShortIndex> list = new ArrayList<ColumnWithShortIndex>(indexes.length / 2);
- list.add(indexes[0]);
+ private void rleEncodeOnData(ColumnWithRowId<Short>[] dataWithRowId) {
+ byte[] prvKey = dataWithRowId[0].getColumn();
+ List<ColumnWithRowId> list = new ArrayList<>(dataWithRowId.length / 2);
+ list.add(dataWithRowId[0]);
short counter = 1;
short start = 0;
List<Short> map = new ArrayList<Short>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
- for (int i = 1; i < indexes.length; i++) {
- if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(prvKey, indexes[i].getColumn()) != 0) {
- prvKey = indexes[i].getColumn();
- list.add(indexes[i]);
+ for (int i = 1; i < dataWithRowId.length; i++) {
+ if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(prvKey, dataWithRowId[i].getColumn()) != 0) {
+ prvKey = dataWithRowId[i].getColumn();
+ list.add(dataWithRowId[i]);
map.add(start);
map.add(counter);
start += counter;
@@ -193,17 +192,17 @@ public class BlockIndexerStorageForShort implements IndexStorage<short[]> {
map.add(counter);
// if rle is index size is more than 70% then rle wont give any benefit
// so better to avoid rle index and write data as it is
- boolean useRle = (((list.size() + map.size()) * 100) / indexes.length) < 70;
+ boolean useRle = (((list.size() + map.size()) * 100) / dataWithRowId.length) < 70;
if (useRle) {
- this.keyBlock = convertToKeyArray(list);
- dataIndexMap = convertToArray(map);
+ this.dataPage = convertToDataPage(list);
+ dataRlePage = convertToArray(map);
} else {
- this.keyBlock = convertToKeyArray(indexes);
- dataIndexMap = new short[0];
+ this.dataPage = convertToDataPage(dataWithRowId);
+ dataRlePage = new short[0];
}
}
- private byte[][] convertToKeyArray(ColumnWithShortIndex[] indexes) {
+ private byte[][] convertToDataPage(ColumnWithRowId[] indexes) {
byte[][] shortArray = new byte[indexes.length][];
for (int i = 0; i < shortArray.length; i++) {
shortArray[i] = indexes[i].getColumn();
@@ -212,7 +211,7 @@ public class BlockIndexerStorageForShort implements IndexStorage<short[]> {
return shortArray;
}
- private byte[][] convertToKeyArray(List<ColumnWithShortIndex> list) {
+ private byte[][] convertToDataPage(List<ColumnWithRowId> list) {
byte[][] shortArray = new byte[list.size()][];
for (int i = 0; i < shortArray.length; i++) {
shortArray[i] = list.get(i).getColumn();
@@ -221,8 +220,8 @@ public class BlockIndexerStorageForShort implements IndexStorage<short[]> {
return shortArray;
}
- @Override public short[] getDataIndexMap() {
- return dataIndexMap;
+ public short[] getDataRlePage() {
+ return dataRlePage;
}
@Override public int getTotalSize() {
@@ -230,11 +229,11 @@ public class BlockIndexerStorageForShort implements IndexStorage<short[]> {
}
@Override public byte[] getMin() {
- return keyBlock[0];
+ return dataPage[0];
}
@Override public byte[] getMax() {
- return keyBlock[keyBlock.length - 1];
+ return dataPage[dataPage.length - 1];
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6a09ee84/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndex.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndex.java
deleted file mode 100644
index fa52e09..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndex.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.datastore.columnar;
-
-import java.util.Arrays;
-
-import org.apache.carbondata.core.util.ByteUtil;
-
-public class ColumnWithIntIndex implements Comparable<ColumnWithIntIndex> {
- protected byte[] column;
-
- private int index;
-
- public ColumnWithIntIndex(byte[] column, int index) {
- this.column = column;
- this.index = index;
- }
-
- public ColumnWithIntIndex() {
- }
-
- /**
- * @return the column
- */
- public byte[] getColumn() {
- return column;
- }
-
- /**
- * @param column the column to set
- */
- public void setColumn(byte[] column) {
- this.column = column;
- }
-
- /**
- * @return the index
- */
- public int getIndex() {
- return index;
- }
-
- /**
- * @param index the index to set
- */
- public void setIndex(int index) {
- this.index = index;
- }
-
- @Override public int compareTo(ColumnWithIntIndex o) {
- return ByteUtil.UnsafeComparer.INSTANCE.compareTo(column, o.column);
- }
-
- @Override public boolean equals(Object obj) {
- if (obj == null || getClass() != obj.getClass()) {
- return false;
- }
- ColumnWithIntIndex o = (ColumnWithIntIndex)obj;
- return Arrays.equals(column, o.column) && index == o.index;
- }
-
- @Override public int hashCode() {
- return Arrays.hashCode(column) + index;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6a09ee84/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndexForHighCard.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndexForHighCard.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndexForHighCard.java
deleted file mode 100644
index 73f59d7..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndexForHighCard.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.datastore.columnar;
-
-import java.util.Arrays;
-
-import org.apache.carbondata.core.util.ByteUtil.UnsafeComparer;
-
-public class ColumnWithIntIndexForHighCard extends ColumnWithIntIndex
- implements Comparable<ColumnWithIntIndex> {
-
- public ColumnWithIntIndexForHighCard(byte[] column, int index) {
- super(column, index);
- }
-
- @Override public int compareTo(ColumnWithIntIndex o) {
- return UnsafeComparer.INSTANCE
- .compareTo(column, 2, column.length - 2, o.column, 2, o.column.length - 2);
- }
-
- @Override public boolean equals(Object obj) {
- if (obj == null || getClass() != obj.getClass()) {
- return false;
- }
- ColumnWithIntIndexForHighCard o = (ColumnWithIntIndexForHighCard)obj;
- return Arrays.equals(column, o.column) && getIndex() == o.getIndex();
- }
-
- @Override public int hashCode() {
- return Arrays.hashCode(column) + getIndex();
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6a09ee84/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithRowId.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithRowId.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithRowId.java
new file mode 100644
index 0000000..dd5d08d
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithRowId.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.datastore.columnar;
+
+import java.util.Arrays;
+
+import org.apache.carbondata.core.util.ByteUtil;
+
+public class ColumnWithRowId<T> implements Comparable<ColumnWithRowId<T>> {
+ protected byte[] column;
+
+ private T index;
+
+ ColumnWithRowId(byte[] column, T index) {
+ this.column = column;
+ this.index = index;
+ }
+
+ /**
+ * @return the column
+ */
+ public byte[] getColumn() {
+ return column;
+ }
+
+ /**
+ * @param column the column to set
+ */
+ public void setColumn(byte[] column) {
+ this.column = column;
+ }
+
+ /**
+ * @return the index
+ */
+ public T getIndex() {
+ return index;
+ }
+
+ /**
+ * @param index the index to set
+ */
+ public void setIndex(T index) {
+ this.index = index;
+ }
+
+ @Override public int compareTo(ColumnWithRowId o) {
+ return ByteUtil.UnsafeComparer.INSTANCE.compareTo(column, o.column);
+ }
+
+ @Override public boolean equals(Object obj) {
+ if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ ColumnWithRowId o = (ColumnWithRowId)obj;
+ return Arrays.equals(column, o.column) && index == o.index;
+ }
+
+ @Override public int hashCode() {
+ return Arrays.hashCode(column) + index.hashCode();
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6a09ee84/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithRowIdForHighCard.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithRowIdForHighCard.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithRowIdForHighCard.java
new file mode 100644
index 0000000..6aef7fd
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithRowIdForHighCard.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.datastore.columnar;
+
+import java.util.Arrays;
+
+import org.apache.carbondata.core.util.ByteUtil.UnsafeComparer;
+
+public class ColumnWithRowIdForHighCard<T> extends ColumnWithRowId<T>
+ implements Comparable<ColumnWithRowId<T>> {
+
+ ColumnWithRowIdForHighCard(byte[] column, T index) {
+ super(column, index);
+ }
+
+ @Override public int compareTo(ColumnWithRowId o) {
+ return UnsafeComparer.INSTANCE
+ .compareTo(column, 2, column.length - 2, o.column, 2, o.column.length - 2);
+ }
+
+ @Override public boolean equals(Object obj) {
+ if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ ColumnWithRowIdForHighCard o = (ColumnWithRowIdForHighCard)obj;
+ return Arrays.equals(column, o.column) && getIndex() == o.getIndex();
+ }
+
+ @Override public int hashCode() {
+ return Arrays.hashCode(column) + getIndex().hashCode();
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6a09ee84/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithShortIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithShortIndex.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithShortIndex.java
deleted file mode 100644
index 57447a3..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithShortIndex.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.datastore.columnar;
-
-import java.util.Arrays;
-
-import org.apache.carbondata.core.util.ByteUtil;
-
-public class ColumnWithShortIndex implements Comparable<ColumnWithShortIndex> {
- protected byte[] column;
-
- private short index;
-
- public ColumnWithShortIndex(byte[] column, short index) {
- this.column = column;
- this.index = index;
- }
-
- /**
- * @return the column
- */
- public byte[] getColumn() {
- return column;
- }
-
- /**
- * @param column the column to set
- */
- public void setColumn(byte[] column) {
- this.column = column;
- }
-
- /**
- * @return the index
- */
- public short getIndex() {
- return index;
- }
-
- /**
- * @param index the index to set
- */
- public void setIndex(short index) {
- this.index = index;
- }
-
- @Override public int compareTo(ColumnWithShortIndex o) {
- return ByteUtil.UnsafeComparer.INSTANCE.compareTo(column, o.column);
- }
-
- @Override public boolean equals(Object obj) {
- if (obj == null || getClass() != obj.getClass()) {
- return false;
- }
- ColumnWithShortIndex o = (ColumnWithShortIndex)obj;
- return Arrays.equals(column, o.column) && index == o.index;
- }
-
- @Override public int hashCode() {
- return Arrays.hashCode(column) + index;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6a09ee84/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithShortIndexForNoDictionay.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithShortIndexForNoDictionay.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithShortIndexForNoDictionay.java
deleted file mode 100644
index 34cce63..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithShortIndexForNoDictionay.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.datastore.columnar;
-
-import java.util.Arrays;
-
-import org.apache.carbondata.core.util.ByteUtil.UnsafeComparer;
-
-public class ColumnWithShortIndexForNoDictionay extends ColumnWithShortIndex
- implements Comparable<ColumnWithShortIndex> {
-
- public ColumnWithShortIndexForNoDictionay(byte[] column, short index) {
- super(column, index);
- }
-
- @Override public int compareTo(ColumnWithShortIndex o) {
- return UnsafeComparer.INSTANCE
- .compareTo(column, 2, column.length - 2, o.column, 2, o.column.length - 2);
- }
-
- @Override public boolean equals(Object obj) {
- if (obj == null || getClass() != obj.getClass()) {
- return false;
- }
- ColumnWithIntIndexForHighCard o = (ColumnWithIntIndexForHighCard) obj;
- return Arrays.equals(column, o.column) && getIndex() == o.getIndex();
- }
-
- @Override public int hashCode() {
- return Arrays.hashCode(column) + getIndex();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6a09ee84/core/src/main/java/org/apache/carbondata/core/datastore/columnar/IndexStorage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/IndexStorage.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/IndexStorage.java
index ff478f3..0570317 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/IndexStorage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/IndexStorage.java
@@ -20,13 +20,13 @@ package org.apache.carbondata.core.datastore.columnar;
public interface IndexStorage<T> {
boolean isAlreadySorted();
- T getDataAfterComp();
+ T getRowIdPage();
- T getIndexMap();
+ T getRowIdRlePage();
- byte[][] getKeyBlock();
+ byte[][] getDataPage();
- T getDataIndexMap();
+ T getDataRlePage();
int getTotalSize();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6a09ee84/processing/src/main/java/org/apache/carbondata/core/datastore/columnar/ColGroupBlockStorage.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/core/datastore/columnar/ColGroupBlockStorage.java b/processing/src/main/java/org/apache/carbondata/core/datastore/columnar/ColGroupBlockStorage.java
index fb8fe3d..70ac3d9 100644
--- a/processing/src/main/java/org/apache/carbondata/core/datastore/columnar/ColGroupBlockStorage.java
+++ b/processing/src/main/java/org/apache/carbondata/core/datastore/columnar/ColGroupBlockStorage.java
@@ -25,19 +25,20 @@ import org.apache.carbondata.processing.store.colgroup.ColGroupDataHolder;
import org.apache.carbondata.processing.store.colgroup.ColGroupMinMax;
/**
- * it is holder of column group data and also min max for colgroup block data
+ * it is holder of column group dataPage and also min max for colgroup block dataPage
*/
public class ColGroupBlockStorage implements IndexStorage, Callable<IndexStorage> {
- private byte[][] data;
+ private byte[][] dataPage;
private ColGroupMinMax colGrpMinMax;
- public ColGroupBlockStorage(SegmentProperties segmentProperties, int colGrpIndex, byte[][] data) {
+ public ColGroupBlockStorage(SegmentProperties segmentProperties, int colGrpIndex,
+ byte[][] dataPage) {
colGrpMinMax = new ColGroupMinMax(segmentProperties, colGrpIndex);
- this.data = data;
- for (int i = 0; i < data.length; i++) {
- colGrpMinMax.add(data[i]);
+ this.dataPage = dataPage;
+ for (int i = 0; i < dataPage.length; i++) {
+ colGrpMinMax.add(dataPage[i]);
}
}
@@ -51,7 +52,7 @@ public class ColGroupBlockStorage implements IndexStorage, Callable<IndexStorage
/**
* for column group storage its not required
*/
- @Override public ColGroupDataHolder getDataAfterComp() {
+ public ColGroupDataHolder getRowIdPage() {
//not required for column group storage
return null;
}
@@ -59,7 +60,7 @@ public class ColGroupBlockStorage implements IndexStorage, Callable<IndexStorage
/**
* for column group storage its not required
*/
- @Override public ColGroupDataHolder getIndexMap() {
+ public ColGroupDataHolder getRowIdRlePage() {
// not required for column group storage
return null;
}
@@ -67,14 +68,14 @@ public class ColGroupBlockStorage implements IndexStorage, Callable<IndexStorage
/**
* for column group storage its not required
*/
- @Override public byte[][] getKeyBlock() {
- return data;
+ public byte[][] getDataPage() {
+ return dataPage;
}
/**
* for column group storage its not required
*/
- @Override public ColGroupDataHolder getDataIndexMap() {
+ public ColGroupDataHolder getDataRlePage() {
//not required for column group
return null;
}
@@ -83,7 +84,7 @@ public class ColGroupBlockStorage implements IndexStorage, Callable<IndexStorage
* for column group storage its not required
*/
@Override public int getTotalSize() {
- return data.length;
+ return dataPage.length;
}
@Override public byte[] getMin() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6a09ee84/processing/src/main/java/org/apache/carbondata/processing/store/TablePageEncoder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/TablePageEncoder.java b/processing/src/main/java/org/apache/carbondata/processing/store/TablePageEncoder.java
index 5d460a9..18c2297 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/TablePageEncoder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/TablePageEncoder.java
@@ -21,7 +21,7 @@ import java.util.Iterator;
import org.apache.carbondata.core.datastore.TableSpec;
import org.apache.carbondata.core.datastore.columnar.BlockIndexerStorageForInt;
-import org.apache.carbondata.core.datastore.columnar.BlockIndexerStorageForNoInvertedIndex;
+import org.apache.carbondata.core.datastore.columnar.BlockIndexerStorageForNoInvertedIndexForInt;
import org.apache.carbondata.core.datastore.columnar.BlockIndexerStorageForNoInvertedIndexForShort;
import org.apache.carbondata.core.datastore.columnar.BlockIndexerStorageForShort;
import org.apache.carbondata.core.datastore.columnar.IndexStorage;
@@ -84,7 +84,7 @@ public class TablePageEncoder {
if (version == ColumnarFormatVersion.V3) {
return new BlockIndexerStorageForNoInvertedIndexForShort(data, false);
} else {
- return new BlockIndexerStorageForNoInvertedIndex(data);
+ return new BlockIndexerStorageForNoInvertedIndexForInt(data);
}
}
}
@@ -101,7 +101,7 @@ public class TablePageEncoder {
if (version == ColumnarFormatVersion.V3) {
return new BlockIndexerStorageForNoInvertedIndexForShort(data, false);
} else {
- return new BlockIndexerStorageForNoInvertedIndex(data);
+ return new BlockIndexerStorageForNoInvertedIndexForInt(data);
}
}
}
@@ -126,7 +126,7 @@ public class TablePageEncoder {
if (version == ColumnarFormatVersion.V3) {
return new BlockIndexerStorageForNoInvertedIndexForShort(data, true);
} else {
- return new BlockIndexerStorageForNoInvertedIndex(data);
+ return new BlockIndexerStorageForNoInvertedIndexForInt(data);
}
}
}
@@ -152,7 +152,7 @@ public class TablePageEncoder {
tablePage.getDictDimensionPage()[++dictionaryColumnCount].getByteArrayPage(),
isSortColumn,
isUseInvertedIndex[i] & isSortColumn);
- flattened = ByteUtil.flatten(indexStorages[indexStorageOffset].getKeyBlock());
+ flattened = ByteUtil.flatten(indexStorages[indexStorageOffset].getDataPage());
break;
case DIRECT_DICTIONARY:
// timestamp and date column
@@ -161,7 +161,7 @@ public class TablePageEncoder {
tablePage.getDictDimensionPage()[++dictionaryColumnCount].getByteArrayPage(),
isSortColumn,
isUseInvertedIndex[i] & isSortColumn);
- flattened = ByteUtil.flatten(indexStorages[indexStorageOffset].getKeyBlock());
+ flattened = ByteUtil.flatten(indexStorages[indexStorageOffset].getDataPage());
break;
case PLAIN_VALUE:
// high cardinality dimension, encoded as plain string
@@ -170,7 +170,7 @@ public class TablePageEncoder {
tablePage.getNoDictDimensionPage()[++noDictionaryColumnCount].getByteArrayPage(),
isSortColumn,
isUseInvertedIndex[i] & isSortColumn);
- flattened = ByteUtil.flatten(indexStorages[indexStorageOffset].getKeyBlock());
+ flattened = ByteUtil.flatten(indexStorages[indexStorageOffset].getDataPage());
break;
case COMPLEX:
// we need to add complex column at last, so skipping it here
@@ -188,7 +188,7 @@ public class TablePageEncoder {
while (iterator.hasNext()) {
byte[][] data = iterator.next();
indexStorages[indexStorageOffset] = encodeAndCompressComplexDimension(data);
- byte[] flattened = ByteUtil.flatten(indexStorages[indexStorageOffset].getKeyBlock());
+ byte[] flattened = ByteUtil.flatten(indexStorages[indexStorageOffset].getDataPage());
compressedColumns[indexStorageOffset] = compressor.compressByte(flattened);
indexStorageOffset++;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6a09ee84/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java
index bf35cd9..fab1a39 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java
@@ -106,10 +106,10 @@ public class CarbonFactDataWriterImplV1 extends AbstractFactDataWriter<int[]> {
for (int i = 0; i < isSortedData.length; i++) {
if (!isSortedData[i]) {
dataAfterCompression[idx] =
- numberCompressor.compress((int[])keyStorageArray[i].getDataAfterComp());
- if (null != keyStorageArray[i].getIndexMap()
- && ((int[])keyStorageArray[i].getIndexMap()).length > 0) {
- indexMap[idx] = numberCompressor.compress((int[])keyStorageArray[i].getIndexMap());
+ numberCompressor.compress((int[])keyStorageArray[i].getRowIdPage());
+ if (null != keyStorageArray[i].getRowIdRlePage()
+ && ((int[])keyStorageArray[i].getRowIdRlePage()).length > 0) {
+ indexMap[idx] = numberCompressor.compress((int[])keyStorageArray[i].getRowIdRlePage());
} else {
indexMap[idx] = new byte[0];
}
@@ -131,7 +131,7 @@ public class CarbonFactDataWriterImplV1 extends AbstractFactDataWriter<int[]> {
if (dataWriterVo.getRleEncodingForDictDim()[i]) {
try {
compressedDataIndex[idx] =
- numberCompressor.compress((int[])keyStorageArray[i].getDataIndexMap());
+ numberCompressor.compress((int[])keyStorageArray[i].getDataRlePage());
dataIndexMapLength[idx] = compressedDataIndex[idx].length;
idx++;
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6a09ee84/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
index b4bc040..a63e902 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
@@ -114,10 +114,10 @@ public class CarbonFactDataWriterImplV3 extends AbstractFactDataWriter<short[]>
totalKeySize += keyLengths[i];
if (!isSortedData[i]) {
dataAfterCompression[i] =
- getByteArray((short[])encoded.indexStorages[i].getDataAfterComp());
- if (null != encoded.indexStorages[i].getIndexMap() &&
- ((short[])encoded.indexStorages[i].getIndexMap()).length > 0) {
- indexMap[i] = getByteArray((short[])encoded.indexStorages[i].getIndexMap());
+ getByteArray((short[])encoded.indexStorages[i].getRowIdPage());
+ if (null != encoded.indexStorages[i].getRowIdRlePage() &&
+ ((short[])encoded.indexStorages[i].getRowIdRlePage()).length > 0) {
+ indexMap[i] = getByteArray((short[])encoded.indexStorages[i].getRowIdRlePage());
} else {
indexMap[i] = new byte[0];
}
@@ -135,7 +135,7 @@ public class CarbonFactDataWriterImplV3 extends AbstractFactDataWriter<short[]>
if (dataWriterVo.getRleEncodingForDictDim()[i]) {
try {
compressedDataIndex[i] =
- getByteArray((short[])encoded.indexStorages[i].getDataIndexMap());
+ getByteArray((short[])encoded.indexStorages[i].getDataRlePage());
dataIndexMapLength[i] = compressedDataIndex[i].length;
} catch (Exception e) {
throw new CarbonDataWriterException(e.getMessage(), e);