You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ch...@apache.org on 2017/02/28 13:58:38 UTC
[1/2] incubator-carbondata git commit: FixedTestcasefailureIssue
Repository: incubator-carbondata
Updated Branches:
refs/heads/master 891fa4346 -> f4e3ec819
FixedTestcasefailureIssue
Fixed failure for V1 format
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/b16c3088
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/b16c3088
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/b16c3088
Branch: refs/heads/master
Commit: b16c30885e706764bbcd87adaf850a88b1411698
Parents: 891fa43
Author: kumarvishal <ku...@gmail.com>
Authored: Tue Feb 28 19:21:50 2017 +0800
Committer: chenliang613 <ch...@huawei.com>
Committed: Tue Feb 28 21:57:05 2017 +0800
----------------------------------------------------------------------
.../BlockIndexerStorageForNoInvertedIndex.java | 135 ++++++-------------
...ndexerStorageForNoInvertedIndexForShort.java | 104 ++++++++++++++
.../store/CarbonFactDataHandlerColumnar.java | 8 +-
3 files changed, 151 insertions(+), 96 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b16c3088/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java
index d9875a0..0ef2518 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java
@@ -14,128 +14,76 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.carbondata.core.datastore.columnar;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.List;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.columnar.IndexStorage;
import org.apache.carbondata.core.util.ByteUtil;
+/**
+ * Below class will be used to for no inverted index
+ */
public class BlockIndexerStorageForNoInvertedIndex implements IndexStorage<int[]> {
+
+ /**
+ * column data
+ */
private byte[][] keyBlock;
- private byte[][] sortedBlock;
+
+ /**
+ * total number of rows
+ */
private int totalSize;
- private int[] dataIndexMap;
- public BlockIndexerStorageForNoInvertedIndex(byte[][] keyBlockInput, boolean compressData,
- boolean isNoDictionary) {
- // without invertedindex but can be RLE
- if (compressData) {
- // with RLE
- byte[] prvKey = keyBlockInput[0];
- List<byte[]> list = new ArrayList<byte[]>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
- list.add(keyBlockInput[0]);
- int counter = 1;
- int start = 0;
- List<Integer> map = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
- int length = keyBlockInput.length;
- for (int i = 1; i < length; i++) {
- if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(prvKey, keyBlockInput[i]) != 0) {
- prvKey = keyBlockInput[i];
- list.add(keyBlockInput[i]);
- map.add(start);
- map.add(counter);
- start += counter;
- counter = 1;
- continue;
- }
- counter++;
+ private byte[] min;
+ private byte[] max;
+
+ public BlockIndexerStorageForNoInvertedIndex(byte[][] keyBlockInput, boolean isNoDictionary) {
+ this.keyBlock = keyBlockInput;
+ min = keyBlock[0];
+ max = keyBlock[0];
+ totalSize += keyBlock[0].length;
+ int minCompare = 0;
+ int maxCompare = 0;
+ for (int i = 1; i < keyBlock.length; i++) {
+ totalSize += keyBlock[i].length;
+ minCompare = ByteUtil.compare(min, keyBlock[i]);
+ maxCompare = ByteUtil.compare(max, keyBlock[i]);
+ if (minCompare > 0) {
+ min = keyBlock[i];
}
- map.add(start);
- map.add(counter);
- this.keyBlock = convertToKeyArray(list);
- if (keyBlockInput.length == this.keyBlock.length) {
- dataIndexMap = new int[0];
- } else {
- dataIndexMap = convertToArray(map);
+ if (maxCompare < 0) {
+ max = keyBlock[i];
}
- } else {
- this.keyBlock = keyBlockInput;
- dataIndexMap = new int[0];
- }
-
- this.sortedBlock = new byte[keyBlock.length][];
- System.arraycopy(keyBlock, 0, sortedBlock, 0, keyBlock.length);
- if (isNoDictionary) {
- Arrays.sort(sortedBlock, new Comparator<byte[]>() {
- @Override
- public int compare(byte[] col1, byte[] col2) {
- return ByteUtil.UnsafeComparer.INSTANCE
- .compareTo(col1, 2, col1.length - 2, col2, 2, col2.length - 2);
- }
- });
- } else {
- Arrays.sort(sortedBlock, new Comparator<byte[]>() {
- @Override
- public int compare(byte[] col1, byte[] col2) {
- return ByteUtil.UnsafeComparer.INSTANCE.compareTo(col1, col2);
- }
- });
- }
-
- }
-
- private int[] convertToArray(List<Integer> list) {
- int[] shortArray = new int[list.size()];
- for (int i = 0; i < shortArray.length; i++) {
- shortArray[i] = list.get(i);
}
- return shortArray;
}
- private byte[][] convertToKeyArray(List<byte[]> list) {
- byte[][] shortArray = new byte[list.size()][];
- for (int i = 0; i < shortArray.length; i++) {
- shortArray[i] = list.get(i);
- totalSize += shortArray[i].length;
- }
- return shortArray;
- }
-
- @Override
- public int[] getDataIndexMap() {
- return dataIndexMap;
+ @Override public int[] getDataIndexMap() {
+ return new int[0];
}
- @Override
- public int getTotalSize() {
+ @Override public int getTotalSize() {
return totalSize;
}
- @Override
- public boolean isAlreadySorted() {
+ @Override public boolean isAlreadySorted() {
return true;
}
/**
* no use
+ *
* @return
*/
- @Override
- public int[] getDataAfterComp() {
+ @Override public int[] getDataAfterComp() {
return new int[0];
}
/**
* no use
+ *
* @return
*/
- @Override
- public int[] getIndexMap() {
+ @Override public int[] getIndexMap() {
return new int[0];
}
@@ -147,11 +95,10 @@ public class BlockIndexerStorageForNoInvertedIndex implements IndexStorage<int[]
}
@Override public byte[] getMin() {
- return sortedBlock[0];
+ return min;
}
@Override public byte[] getMax() {
- return sortedBlock[sortedBlock.length - 1];
+ return max;
}
-
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b16c3088/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndexForShort.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndexForShort.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndexForShort.java
new file mode 100644
index 0000000..731df96
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndexForShort.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datastore.columnar;
+
+import org.apache.carbondata.core.util.ByteUtil;
+
+/**
+ * Below class will be used to for no inverted index
+ */
+public class BlockIndexerStorageForNoInvertedIndexForShort implements IndexStorage<short[]> {
+
+ /**
+ * column data
+ */
+ private byte[][] keyBlock;
+
+ /**
+ * total number of rows
+ */
+ private int totalSize;
+
+ private byte[] min;
+ private byte[] max;
+
+ public BlockIndexerStorageForNoInvertedIndexForShort(byte[][] keyBlockInput,
+ boolean isNoDictionary) {
+ this.keyBlock = keyBlockInput;
+ min = keyBlock[0];
+ max = keyBlock[0];
+ totalSize += keyBlock[0].length;
+ int minCompare = 0;
+ int maxCompare = 0;
+ for (int i = 1; i < keyBlock.length; i++) {
+ totalSize += keyBlock[i].length;
+ minCompare = ByteUtil.compare(min, keyBlock[i]);
+ maxCompare = ByteUtil.compare(max, keyBlock[i]);
+ if (minCompare > 0) {
+ min = keyBlock[i];
+ }
+ if (maxCompare < 0) {
+ max = keyBlock[i];
+ }
+ }
+ }
+
+ @Override public short[] getDataIndexMap() {
+ return new short[0];
+ }
+
+ @Override public int getTotalSize() {
+ return totalSize;
+ }
+
+ @Override public boolean isAlreadySorted() {
+ return true;
+ }
+
+ /**
+ * no use
+ *
+ * @return
+ */
+ @Override public short[] getDataAfterComp() {
+ return new short[0];
+ }
+
+ /**
+ * no use
+ *
+ * @return
+ */
+ @Override public short[] getIndexMap() {
+ return new short[0];
+ }
+
+ /**
+ * @return the keyBlock
+ */
+ public byte[][] getKeyBlock() {
+ return keyBlock;
+ }
+
+ @Override public byte[] getMin() {
+ return min;
+ }
+
+ @Override public byte[] getMax() {
+ return max;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b16c3088/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
index 0699167..ed461f9 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
@@ -44,6 +44,7 @@ import org.apache.carbondata.core.constants.CarbonV3DataFormatConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.columnar.BlockIndexerStorageForInt;
import org.apache.carbondata.core.datastore.columnar.BlockIndexerStorageForNoInvertedIndex;
+import org.apache.carbondata.core.datastore.columnar.BlockIndexerStorageForNoInvertedIndexForShort;
import org.apache.carbondata.core.datastore.columnar.BlockIndexerStorageForShort;
import org.apache.carbondata.core.datastore.columnar.ColumnGroupModel;
import org.apache.carbondata.core.datastore.columnar.IndexStorage;
@@ -1637,8 +1638,11 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
isSortRequired);
}
} else {
- return new BlockIndexerStorageForNoInvertedIndex(this.data, isCompressionReq,
- isNoDictionary);
+ if (version == ColumnarFormatVersion.V3) {
+ return new BlockIndexerStorageForNoInvertedIndexForShort(this.data, isNoDictionary);
+ } else {
+ return new BlockIndexerStorageForNoInvertedIndex(this.data, isNoDictionary);
+ }
}
}
[2/2] incubator-carbondata git commit: [CARBONDATA-733]Fixed Testcase
failure Issue This closes #615
Posted by ch...@apache.org.
[CARBONDATA-733]Fixed Testcase failure Issue This closes #615
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/f4e3ec81
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/f4e3ec81
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/f4e3ec81
Branch: refs/heads/master
Commit: f4e3ec81952195f0f1e7ecb077870ac2d8a88ed3
Parents: 891fa43 b16c308
Author: chenliang613 <ch...@huawei.com>
Authored: Tue Feb 28 21:58:17 2017 +0800
Committer: chenliang613 <ch...@huawei.com>
Committed: Tue Feb 28 21:58:17 2017 +0800
----------------------------------------------------------------------
.../BlockIndexerStorageForNoInvertedIndex.java | 135 ++++++-------------
...ndexerStorageForNoInvertedIndexForShort.java | 104 ++++++++++++++
.../store/CarbonFactDataHandlerColumnar.java | 8 +-
3 files changed, 151 insertions(+), 96 deletions(-)
----------------------------------------------------------------------