You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2017/05/15 05:06:14 UTC
[01/19] carbondata git commit: [CARBONDATA-782] add SORT_COLUMNS
options
Repository: carbondata
Updated Branches:
refs/heads/master fb2441a93 -> 46521add0
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/NoSortProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/NoSortProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/NoSortProcessorStepImpl.java
new file mode 100644
index 0000000..bde89ed
--- /dev/null
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/NoSortProcessorStepImpl.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.processing.newflow.steps;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.util.DataTypeUtil;
+import org.apache.carbondata.processing.newflow.AbstractDataLoadProcessorStep;
+import org.apache.carbondata.processing.newflow.CarbonDataLoadConfiguration;
+import org.apache.carbondata.processing.newflow.DataField;
+import org.apache.carbondata.processing.newflow.row.CarbonRow;
+import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
+import org.apache.carbondata.processing.util.NonDictionaryUtil;
+
+/**
+ * if the table doesn't have sort_columns, just convert row format.
+ */
+public class NoSortProcessorStepImpl extends AbstractDataLoadProcessorStep {
+
+ private int dimensionCount;
+
+ private int dimensionWithComplexCount;
+
+ private int noDictCount;
+
+ private int measureCount;
+
+ private boolean[] isNoDictionaryDimensionColumn;
+
+ private char[] aggType;
+
+ public NoSortProcessorStepImpl(CarbonDataLoadConfiguration configuration,
+ AbstractDataLoadProcessorStep child) {
+ super(configuration, child);
+ this.dimensionWithComplexCount = configuration.getDimensionCount();
+ this.noDictCount =
+ configuration.getNoDictionaryCount() + configuration.getComplexDimensionCount();
+ this.dimensionCount = configuration.getDimensionCount() - this.noDictCount;
+ this.measureCount = configuration.getMeasureCount();
+ this.isNoDictionaryDimensionColumn =
+ CarbonDataProcessorUtil.getNoDictionaryMapping(configuration.getDataFields());
+ this.aggType = CarbonDataProcessorUtil
+ .getAggType(configuration.getMeasureCount(), configuration.getMeasureFields());
+ }
+
+ @Override public DataField[] getOutput() {
+ return child.getOutput();
+ }
+
+ @Override public void initialize() throws IOException {
+ child.initialize();
+ }
+
+ /**
+ * convert input CarbonRow to output CarbonRow
+ * e.g. There is a table as following,
+ * the number of dictionary dimensions is a,
+ * the number of no-dictionary dimensions is b,
+ * the number of complex dimensions is c,
+ * the number of measures is d.
+ * input CarbonRow format: the length of Object[] data is a+b+c+d, the number of all columns.
+ * ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ * | Part | Object item | describe |
+ * ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ * | Object[0 ~ a+b-1] | Integer, byte[], Integer, ... | dict + no dict dimensions|
+ * ----------------------------------------------------------------------------------------
+ * | Object[a+b ~ a+b+c-1] | byte[], byte[], ... | complex dimensions |
+ * ----------------------------------------------------------------------------------------
+ * | Object[a+b+c ~ a+b+c+d-1]| int, byte[], ... | measures |
+ * ----------------------------------------------------------------------------------------
+ * output CarbonRow format: the length of object[] data is 3.
+ * ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ * | Part | Object item | describe |
+ * ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ * | Object[0] | int[a] | dict dimension array |
+ * ----------------------------------------------------------------------------------------
+ * | Object[1] | byte[b+c][] | no dict + complex dim |
+ * ----------------------------------------------------------------------------------------
+ * | Object[2] | Object[d] | measures |
+ * ----------------------------------------------------------------------------------------
+ * @param row
+ * @return
+ */
+ @Override protected CarbonRow processRow(CarbonRow row) {
+ int dictIndex = 0;
+ int nonDicIndex = 0;
+ int[] dim = new int[this.dimensionCount];
+ byte[][] nonDicArray = new byte[this.noDictCount][];
+ Object[] measures = new Object[this.measureCount];
+ // read dimension values
+ int dimCount = 0;
+ for (; dimCount < isNoDictionaryDimensionColumn.length; dimCount++) {
+ if (isNoDictionaryDimensionColumn[dimCount]) {
+ nonDicArray[nonDicIndex++] = (byte[]) row.getObject(dimCount);
+ } else {
+ dim[dictIndex++] = (int) row.getObject(dimCount);
+ }
+ }
+
+ for (; dimCount < this.dimensionWithComplexCount; dimCount++) {
+ nonDicArray[nonDicIndex++] = (byte[]) row.getObject(dimCount);
+ }
+
+ // measure values
+ for (int mesCount = 0; mesCount < this.measureCount; mesCount++) {
+ Object value = row.getObject(mesCount + this.dimensionWithComplexCount);
+ if (null != value) {
+ if (aggType[mesCount] == CarbonCommonConstants.DOUBLE_MEASURE) {
+ measures[mesCount] = value;
+ } else if (aggType[mesCount] == CarbonCommonConstants.BIG_INT_MEASURE) {
+ measures[mesCount] = value;
+ } else if (aggType[mesCount] == CarbonCommonConstants.BIG_DECIMAL_MEASURE) {
+ BigDecimal val = (BigDecimal) value;
+ measures[mesCount] = DataTypeUtil.bigDecimalToByte(val);
+ }
+ } else {
+ measures[mesCount] = null;
+ }
+ }
+ // create new row of size 3 (1 for dims , 1 for high card , 1 for measures)
+ Object[] holder = new Object[3];
+ NonDictionaryUtil.prepareOutObj(holder, dim, nonDicArray, measures);
+ //return out row
+ return new CarbonRow(holder);
+ }
+
+ @Override
+ public void close() {
+ if (!closed) {
+ super.close();
+ }
+ }
+
+ @Override protected String getStepName() {
+ return "No Sort Processor";
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/IntermediateFileMerger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/IntermediateFileMerger.java b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/IntermediateFileMerger.java
index 5487593..a9e762d 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/IntermediateFileMerger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/IntermediateFileMerger.java
@@ -251,7 +251,8 @@ public class IntermediateFileMerger implements Callable<Void> {
new SortTempFileChunkHolder(tempFile, mergerParameters.getDimColCount(),
mergerParameters.getComplexDimColCount(), mergerParameters.getMeasureColCount(),
mergerParameters.getFileBufferSize(), mergerParameters.getNoDictionaryCount(),
- mergerParameters.getAggType(), mergerParameters.getNoDictionaryDimnesionColumn());
+ mergerParameters.getAggType(), mergerParameters.getNoDictionaryDimnesionColumn(),
+ mergerParameters.getNoDictionarySortColumn());
// initialize
sortTempFileChunkHolder.initialize();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/NewRowComparator.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/NewRowComparator.java b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/NewRowComparator.java
index dd9358c..247251e 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/NewRowComparator.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/NewRowComparator.java
@@ -24,15 +24,15 @@ import org.apache.carbondata.core.util.ByteUtil.UnsafeComparer;
public class NewRowComparator implements Comparator<Object[]> {
/**
- * noDictionaryColMaping mapping of dictionary dimensions and no dictionary dimensions.
+ * mapping of dictionary dimensions and no dictionary of sort_column.
*/
- private boolean[] noDictionaryColMaping;
+ private boolean[] noDictionarySortColumnMaping;
/**
- * @param noDictionaryColMaping
+ * @param noDictionarySortColumnMaping
*/
- public NewRowComparator(boolean[] noDictionaryColMaping) {
- this.noDictionaryColMaping = noDictionaryColMaping;
+ public NewRowComparator(boolean[] noDictionarySortColumnMaping) {
+ this.noDictionarySortColumnMaping = noDictionarySortColumnMaping;
}
/**
@@ -43,7 +43,7 @@ public class NewRowComparator implements Comparator<Object[]> {
int index = 0;
- for (boolean isNoDictionary : noDictionaryColMaping) {
+ for (boolean isNoDictionary : noDictionarySortColumnMaping) {
if (isNoDictionary) {
byte[] byteArr1 = (byte[]) rowA[index];
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/NewRowComparatorForNormalDims.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/NewRowComparatorForNormalDims.java b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/NewRowComparatorForNormalDims.java
index d913b32..241882e 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/NewRowComparatorForNormalDims.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/NewRowComparatorForNormalDims.java
@@ -26,15 +26,15 @@ public class NewRowComparatorForNormalDims implements Comparator<Object[]> {
/**
* dimension count
*/
- private int dimensionCount;
+ private int numberOfSortColumns;
/**
* RowComparatorForNormalDims Constructor
*
- * @param dimensionCount
+ * @param numberOfSortColumns
*/
- public NewRowComparatorForNormalDims(int dimensionCount) {
- this.dimensionCount = dimensionCount;
+ public NewRowComparatorForNormalDims(int numberOfSortColumns) {
+ this.numberOfSortColumns = numberOfSortColumns;
}
/**
@@ -45,7 +45,7 @@ public class NewRowComparatorForNormalDims implements Comparator<Object[]> {
public int compare(Object[] rowA, Object[] rowB) {
int diff = 0;
- for (int i = 0; i < dimensionCount; i++) {
+ for (int i = 0; i < numberOfSortColumns; i++) {
int dimFieldA = (int)rowA[i];
int dimFieldB = (int)rowB[i];
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/RowComparator.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/RowComparator.java b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/RowComparator.java
index c282f52..2584048 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/RowComparator.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/RowComparator.java
@@ -33,15 +33,15 @@ public class RowComparator implements Comparator<Object[]> {
/**
* noDictionaryColMaping mapping of dictionary dimensions and no dictionary dimensions.
*/
- private boolean[] noDictionaryColMaping;
+ private boolean[] noDictionarySortColumnMaping;
/**
- * @param noDictionaryColMaping
+ * @param noDictionarySortColumnMaping
* @param noDictionaryCount
*/
- public RowComparator(boolean[] noDictionaryColMaping, int noDictionaryCount) {
+ public RowComparator(boolean[] noDictionarySortColumnMaping, int noDictionaryCount) {
this.noDictionaryCount = noDictionaryCount;
- this.noDictionaryColMaping = noDictionaryColMaping;
+ this.noDictionarySortColumnMaping = noDictionarySortColumnMaping;
}
/**
@@ -53,7 +53,7 @@ public class RowComparator implements Comparator<Object[]> {
int normalIndex = 0;
int noDictionaryindex = 0;
- for (boolean isNoDictionary : noDictionaryColMaping) {
+ for (boolean isNoDictionary : noDictionarySortColumnMaping) {
if (isNoDictionary) {
byte[] byteArr1 = (byte[]) rowA[IgnoreDictionary.BYTE_ARRAY_INDEX_IN_ROW.getIndex()];
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/RowComparatorForNormalDims.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/RowComparatorForNormalDims.java b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/RowComparatorForNormalDims.java
index ceaf5c6..8d914ea 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/RowComparatorForNormalDims.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/RowComparatorForNormalDims.java
@@ -28,15 +28,15 @@ public class RowComparatorForNormalDims implements Comparator<Object[]> {
/**
* dimension count
*/
- private int dimensionCount;
+ private int numberOfSortColumns;
/**
* RowComparatorForNormalDims Constructor
*
- * @param dimensionCount
+ * @param numberOfSortColumns
*/
- public RowComparatorForNormalDims(int dimensionCount) {
- this.dimensionCount = dimensionCount;
+ public RowComparatorForNormalDims(int numberOfSortColumns) {
+ this.numberOfSortColumns = numberOfSortColumns;
}
/**
@@ -47,7 +47,7 @@ public class RowComparatorForNormalDims implements Comparator<Object[]> {
public int compare(Object[] rowA, Object[] rowB) {
int diff = 0;
- for (int i = 0; i < dimensionCount; i++) {
+ for (int i = 0; i < numberOfSortColumns; i++) {
int dimFieldA = NonDictionaryUtil.getDimension(i, rowA);
int dimFieldB = NonDictionaryUtil.getDimension(i, rowB);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortDataRows.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortDataRows.java b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortDataRows.java
index ffaa566..eba5433 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortDataRows.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortDataRows.java
@@ -192,11 +192,10 @@ public class SortDataRows {
Object[][] toSort;
toSort = new Object[entryCount][];
System.arraycopy(recordHolderList, 0, toSort, 0, entryCount);
-
- if (parameters.getNoDictionaryCount() > 0) {
- Arrays.sort(toSort, new NewRowComparator(parameters.getNoDictionaryDimnesionColumn()));
+ if (parameters.getNumberOfNoDictSortColumns() > 0) {
+ Arrays.sort(toSort, new NewRowComparator(parameters.getNoDictionarySortColumn()));
} else {
- Arrays.sort(toSort, new NewRowComparatorForNormalDims(parameters.getDimColCount()));
+ Arrays.sort(toSort, new NewRowComparatorForNormalDims(parameters.getNumberOfSortColumns()));
}
recordHolderList = toSort;
@@ -386,12 +385,12 @@ public class SortDataRows {
@Override public Void call() throws Exception {
try {
long startTime = System.currentTimeMillis();
- if (parameters.getNoDictionaryCount() > 0) {
+ if (parameters.getNumberOfNoDictSortColumns() > 0) {
Arrays.sort(recordHolderArray,
- new NewRowComparator(parameters.getNoDictionaryDimnesionColumn()));
+ new NewRowComparator(parameters.getNoDictionarySortColumn()));
} else {
Arrays.sort(recordHolderArray,
- new NewRowComparatorForNormalDims(parameters.getDimColCount()));
+ new NewRowComparatorForNormalDims(parameters.getNumberOfSortColumns()));
}
// create a new file every time
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortParameters.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortParameters.java b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortParameters.java
index 3c3a9d8..7ef8f8e 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortParameters.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortParameters.java
@@ -112,6 +112,12 @@ public class SortParameters {
*/
private boolean[] noDictionaryDimnesionColumn;
+ private boolean[] noDictionarySortColumn;
+
+ private int numberOfSortColumns;
+
+ private int numberOfNoDictSortColumns;
+
private int numberOfCores;
public SortParameters getCopy() {
@@ -137,6 +143,9 @@ public class SortParameters {
parameters.segmentId = segmentId;
parameters.taskNo = taskNo;
parameters.noDictionaryDimnesionColumn = noDictionaryDimnesionColumn;
+ parameters.noDictionarySortColumn = noDictionarySortColumn;
+ parameters.numberOfSortColumns = numberOfSortColumns;
+ parameters.numberOfNoDictSortColumns = numberOfNoDictSortColumns;
parameters.numberOfCores = numberOfCores;
return parameters;
}
@@ -317,6 +326,30 @@ public class SortParameters {
this.numberOfCores = numberOfCores;
}
+ public int getNumberOfSortColumns() {
+ return numberOfSortColumns;
+ }
+
+ public void setNumberOfSortColumns(int numberOfSortColumns) {
+ this.numberOfSortColumns = Math.min(numberOfSortColumns, this.dimColCount);
+ }
+
+ public boolean[] getNoDictionarySortColumn() {
+ return noDictionarySortColumn;
+ }
+
+ public void setNoDictionarySortColumn(boolean[] noDictionarySortColumn) {
+ this.noDictionarySortColumn = noDictionarySortColumn;
+ }
+
+ public int getNumberOfNoDictSortColumns() {
+ return numberOfNoDictSortColumns;
+ }
+
+ public void setNumberOfNoDictSortColumns(int numberOfNoDictSortColumns) {
+ this.numberOfNoDictSortColumns = Math.min(numberOfNoDictSortColumns, noDictionaryCount);
+ }
+
public static SortParameters createSortParameters(CarbonDataLoadConfiguration configuration) {
SortParameters parameters = new SortParameters();
CarbonTableIdentifier tableIdentifier =
@@ -334,6 +367,16 @@ public class SortParameters {
parameters.setComplexDimColCount(configuration.getComplexDimensionCount());
parameters.setNoDictionaryDimnesionColumn(
CarbonDataProcessorUtil.getNoDictionaryMapping(configuration.getDataFields()));
+ parameters.setNumberOfSortColumns(configuration.getNumberOfSortColumns());
+ parameters.setNumberOfNoDictSortColumns(configuration.getNumberOfNoDictSortColumns());
+ if (parameters.getNumberOfSortColumns() == parameters.getNoDictionaryDimnesionColumn().length) {
+ parameters.setNoDictionarySortColumn(parameters.getNoDictionaryDimnesionColumn());
+ } else {
+ boolean[] noDictionarySortColumnTemp = new boolean[parameters.getNumberOfSortColumns()];
+ System.arraycopy(parameters.getNoDictionaryDimnesionColumn(), 0,
+ noDictionarySortColumnTemp, 0, parameters.getNumberOfSortColumns());
+ parameters.setNoDictionarySortColumn(noDictionarySortColumnTemp);
+ }
parameters.setObserver(new SortObserver());
// get sort buffer size
parameters.setSortBufferSize(Integer.parseInt(carbonProperties
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortTempFileChunkHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortTempFileChunkHolder.java b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortTempFileChunkHolder.java
index b4ccc6f..2b0b8ae 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortTempFileChunkHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortTempFileChunkHolder.java
@@ -133,6 +133,11 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
private boolean[] isNoDictionaryDimensionColumn;
/**
+ * to store whether sort column is of dictionary type or not
+ */
+ private boolean[] isNoDictionarySortColumn;
+
+ /**
* Constructor to initialize
*
* @param tempFile
@@ -146,7 +151,7 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
*/
public SortTempFileChunkHolder(File tempFile, int dimensionCount, int complexDimensionCount,
int measureCount, int fileBufferSize, int noDictionaryCount, char[] aggType,
- boolean[] isNoDictionaryDimensionColumn) {
+ boolean[] isNoDictionaryDimensionColumn, boolean[] isNoDictionarySortColumn) {
// set temp file
this.tempFile = tempFile;
@@ -160,7 +165,9 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
this.fileBufferSize = fileBufferSize;
this.executorService = Executors.newFixedThreadPool(1);
this.aggType = aggType;
+
this.isNoDictionaryDimensionColumn = isNoDictionaryDimensionColumn;
+ this.isNoDictionarySortColumn = isNoDictionarySortColumn;
}
/**
@@ -409,7 +416,7 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
int[] rightMdkArray = (int[]) other.returnRow[0];
byte[][] leftNonDictArray = (byte[][]) returnRow[1];
byte[][] rightNonDictArray = (byte[][]) other.returnRow[1];
- for (boolean isNoDictionary : isNoDictionaryDimensionColumn) {
+ for (boolean isNoDictionary : isNoDictionarySortColumn) {
if (isNoDictionary) {
diff = UnsafeComparer.INSTANCE
.compareTo(leftNonDictArray[noDictionaryIndex], rightNonDictArray[noDictionaryIndex]);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
index f6ceb84..6d81d59 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
@@ -781,13 +781,15 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
int i = 0;
int dictionaryColumnCount = -1;
int noDictionaryColumnCount = -1;
+ boolean isSortColumn = false;
for (i = 0; i < dimensionType.length; i++) {
+ isSortColumn = i < segmentProperties.getNumberOfSortColumns();
if (dimensionType[i]) {
dictionaryColumnCount++;
if (colGrpModel.isColumnar(dictionaryColumnCount)) {
submit.add(executorService.submit(
- new BlockSortThread(i, dataHolders[dictionaryColumnCount].getData(), true,
- isUseInvertedIndex[i])));
+ new BlockSortThread(i, dataHolders[dictionaryColumnCount].getData(), isSortColumn,
+ isUseInvertedIndex[i] & isSortColumn)));
} else {
submit.add(
executorService.submit(new ColGroupBlockStorage(dataHolders[dictionaryColumnCount])));
@@ -795,7 +797,7 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
} else {
submit.add(executorService.submit(
new BlockSortThread(i, noDictionaryColumnsData[++noDictionaryColumnCount], false, true,
- true, isUseInvertedIndex[i])));
+ isSortColumn, isUseInvertedIndex[i] & isSortColumn)));
}
}
for (int k = 0; k < complexColCount; k++) {
@@ -819,7 +821,42 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
}
byte[] composedNonDictStartKey = null;
byte[] composedNonDictEndKey = null;
- if (noDictionaryStartKey != null) {
+
+ int numberOfDictSortColumns = segmentProperties.getNumberOfDictSortColumns();
+ // generate start/end key by sort_columns
+ if (numberOfDictSortColumns > 0) {
+ // if sort_columns contain dictionary columns
+ int[] keySize = columnarSplitter.getBlockKeySize();
+ if (keySize.length > numberOfDictSortColumns) {
+ int newMdkLength = 0;
+ for (int index = 0; index < numberOfDictSortColumns; index++) {
+ newMdkLength += keySize[index];
+ }
+ byte[] newStartKeyOfSortKey = new byte[newMdkLength];
+ byte[] newEndKeyOfSortKey = new byte[newMdkLength];
+ System.arraycopy(startkeyLocal, 0, newStartKeyOfSortKey, 0, newMdkLength);
+ System.arraycopy(endKeyLocal, 0, newEndKeyOfSortKey, 0, newMdkLength);
+ startkeyLocal = newStartKeyOfSortKey;
+ endKeyLocal = newEndKeyOfSortKey;
+ }
+ } else {
+ startkeyLocal = new byte[0];
+ endKeyLocal = new byte[0];
+ }
+
+ int numberOfNoDictSortColumns = segmentProperties.getNumberOfNoDictSortColumns();
+ if (numberOfNoDictSortColumns > 0) {
+ // if sort_columns contain no-dictionary columns
+ if (noDictionaryStartKey.length > numberOfNoDictSortColumns) {
+ byte[][] newNoDictionaryStartKey = new byte[numberOfNoDictSortColumns][];
+ byte[][] newNoDictionaryEndKey = new byte[numberOfNoDictSortColumns][];
+ System.arraycopy(noDictionaryStartKey, 0, newNoDictionaryStartKey, 0,
+ numberOfNoDictSortColumns);
+ System
+ .arraycopy(noDictionaryEndKey, 0, newNoDictionaryEndKey, 0, numberOfNoDictSortColumns);
+ noDictionaryStartKey = newNoDictionaryStartKey;
+ noDictionaryEndKey = newNoDictionaryEndKey;
+ }
composedNonDictStartKey =
NonDictionaryUtil.packByteBufferIntoSingleByteArray(noDictionaryStartKey);
composedNonDictEndKey =
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
index 15c75c1..d3b5605 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
@@ -204,6 +204,13 @@ public class CarbonFactDataHandlerModel {
CarbonDataProcessorUtil.getIsUseInvertedIndex(configuration.getDataFields());
int[] dimLensWithComplex = configuration.getCardinalityFinder().getCardinality();
+ if (!configuration.isSortTable()) {
+ for (int i = 0; i < dimLensWithComplex.length; i++) {
+ if (dimLensWithComplex[i] != 0) {
+ dimLensWithComplex[i] = Integer.MAX_VALUE;
+ }
+ }
+ }
List<Integer> dimsLenList = new ArrayList<Integer>();
for (int eachDimLen : dimLensWithComplex) {
if (eachDimLen != 0) dimsLenList.add(eachDimLen);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/processing/src/main/java/org/apache/carbondata/processing/store/SingleThreadFinalSortFilesMerger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/SingleThreadFinalSortFilesMerger.java b/processing/src/main/java/org/apache/carbondata/processing/store/SingleThreadFinalSortFilesMerger.java
index 68f9bd5..f8454f1 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/SingleThreadFinalSortFilesMerger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/SingleThreadFinalSortFilesMerger.java
@@ -101,9 +101,11 @@ public class SingleThreadFinalSortFilesMerger extends CarbonIterator<Object[]> {
*/
private boolean[] isNoDictionaryColumn;
+ private boolean[] isNoDictionarySortColumn;
+
public SingleThreadFinalSortFilesMerger(String tempFileLocation, String tableName,
int dimensionCount, int complexDimensionCount, int measureCount, int noDictionaryCount,
- char[] aggType, boolean[] isNoDictionaryColumn) {
+ char[] aggType, boolean[] isNoDictionaryColumn, boolean[] isNoDictionarySortColumn) {
this.tempFileLocation = tempFileLocation;
this.tableName = tableName;
this.dimensionCount = dimensionCount;
@@ -112,6 +114,7 @@ public class SingleThreadFinalSortFilesMerger extends CarbonIterator<Object[]> {
this.aggType = aggType;
this.noDictionaryCount = noDictionaryCount;
this.isNoDictionaryColumn = isNoDictionaryColumn;
+ this.isNoDictionarySortColumn = isNoDictionarySortColumn;
}
/**
@@ -180,7 +183,8 @@ public class SingleThreadFinalSortFilesMerger extends CarbonIterator<Object[]> {
// create chunk holder
SortTempFileChunkHolder sortTempFileChunkHolder =
new SortTempFileChunkHolder(tempFile, dimensionCount, complexDimensionCount,
- measureCount, fileBufferSize, noDictionaryCount, aggType, isNoDictionaryColumn);
+ measureCount, fileBufferSize, noDictionaryCount, aggType, isNoDictionaryColumn,
+ isNoDictionarySortColumn);
// initialize
sortTempFileChunkHolder.initialize();
[10/19] carbondata git commit: [CARBONDATA-935] Define PartitionInfo
model
Posted by ra...@apache.org.
[CARBONDATA-935] Define PartitionInfo model
fix javastyle error
modify partitioninfo and singlePartition
modify schema.thrift to add partitioningList
modify definition
variable style fix, remove partitionName and some comments
change import order
change comments
change partition info and create partition statistic
add get method in carbontable
remove partition in schema
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/f4d081e8
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/f4d081e8
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/f4d081e8
Branch: refs/heads/master
Commit: f4d081e81714d2d6795af1eab64a68164a4711dd
Parents: bf44c9f
Author: lucao <wh...@gmail.com>
Authored: Tue Apr 18 14:52:18 2017 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Sun May 14 20:40:13 2017 +0800
----------------------------------------------------------------------
.../core/metadata/schema/PartitionInfo.java | 98 ++++++++++++++++++++
.../schema/partition/AbstractPartition.java | 42 +++++++++
.../schema/partition/HashPartition.java | 34 +++++++
.../schema/partition/ListPartition.java | 36 +++++++
.../schema/partition/PartitionType.java | 27 ++++++
.../schema/partition/RangePartition.java | 40 ++++++++
.../core/metadata/schema/table/CarbonTable.java | 20 ++++
.../core/metadata/schema/table/TableSchema.java | 14 +++
.../core/stats/PartitionStatistic.java | 59 ++++++++++++
format/src/main/thrift/schema.thrift | 18 ++++
10 files changed, 388 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f4d081e8/core/src/main/java/org/apache/carbondata/core/metadata/schema/PartitionInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/PartitionInfo.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/PartitionInfo.java
new file mode 100644
index 0000000..86ef3c5
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/PartitionInfo.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.metadata.schema;
+
+import java.io.Serializable;
+import java.util.List;
+
+import org.apache.carbondata.core.metadata.schema.partition.PartitionType;
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
+
+/**
+ * Partition Information of carbon partition table
+ */
+public class PartitionInfo implements Serializable {
+
+ /**
+ * Partition columns
+ */
+ private List<ColumnSchema> columnSchemaList;
+
+ /**
+ * partition type
+ */
+ private PartitionType partitionType;
+
+ /**
+ * Range Partition definition
+ */
+ private List<String> rangeInfo;
+
+ /**
+ * List Partition definition
+ */
+ private List<List<String>> listInfo;
+
+ /**
+ * Hash Partition numbers
+ */
+ private int hashNumber;
+
+ /**
+ * For range partition table
+ * @param columnSchemaList
+ * @param partitionType
+ */
+ public PartitionInfo(List<ColumnSchema> columnSchemaList, PartitionType partitionType) {
+ this.columnSchemaList = columnSchemaList;
+ this.partitionType = partitionType;
+ }
+
+ public List<ColumnSchema> getColumnSchemaList() {
+ return columnSchemaList;
+ }
+
+ public PartitionType getPartitionType() {
+ return partitionType;
+ }
+
+ public void setHashNumber(int hashNumber) {
+ this.hashNumber = hashNumber;
+ }
+
+ public int getHashNumber() {
+ return hashNumber;
+ }
+
+ public void setRangeInfo(List<String> rangeInfo) {
+ this.rangeInfo = rangeInfo;
+ }
+
+ public List<String> getRangeInfo() {
+ return rangeInfo;
+ }
+
+ public void setListInfo(List<List<String>> listInfo) {
+ this.listInfo = listInfo;
+ }
+
+ public List<List<String>> getListInfo() {
+ return listInfo;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f4d081e8/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/AbstractPartition.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/AbstractPartition.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/AbstractPartition.java
new file mode 100644
index 0000000..1d6337c
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/AbstractPartition.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.metadata.schema.partition;
+
+public abstract class AbstractPartition {
+
+ /**
+ * Partition unique identifier
+ */
+ protected int partitionId;
+
+ /**
+ * Total row count of this partition
+ */
+ protected int rowCount;
+
+ public int getPartitionId() {
+ return partitionId;
+ }
+
+ public void setRowCount(int count) {
+ this.rowCount = count;
+ }
+
+ public int getRowCount() {
+ return rowCount;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f4d081e8/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/HashPartition.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/HashPartition.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/HashPartition.java
new file mode 100644
index 0000000..6b43525
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/HashPartition.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.metadata.schema.partition;
+
+public class HashPartition extends AbstractPartition {
+
+ /**
+ * hash value for hash partition table
+ */
+ private int hashValue;
+
+ public HashPartition(int id, int hashValue) {
+ this.partitionId = id;
+ this.hashValue = hashValue;
+ }
+
+ public int getHashValue() {
+ return hashValue;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f4d081e8/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/ListPartition.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/ListPartition.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/ListPartition.java
new file mode 100644
index 0000000..11a396f
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/ListPartition.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.metadata.schema.partition;
+
+import java.util.List;
+
+public class ListPartition extends AbstractPartition {
+
+ /**
+ * value list for list partition table
+ */
+ private List<String> listInfo;
+
+ public ListPartition(int id, List<String> listInfo) {
+ this.partitionId = id;
+ this.listInfo = listInfo;
+ }
+
+ public List<String> getListInfo() {
+ return listInfo;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f4d081e8/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/PartitionType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/PartitionType.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/PartitionType.java
new file mode 100644
index 0000000..9a4cf4a
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/PartitionType.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.metadata.schema.partition;
+
+/**
+ * Partition type supported in carbon
+ */
+public enum PartitionType {
+ RANGE,
+ RANGE_INTERVAL,
+ LIST,
+ HASH
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f4d081e8/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/RangePartition.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/RangePartition.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/RangePartition.java
new file mode 100644
index 0000000..af2b05d
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/RangePartition.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.metadata.schema.partition;
+
+public class RangePartition extends AbstractPartition {
+ /**
+ * boundary value for range partition table
+ */
+ private String lowerBoundary;
+
+ private String upperBoundary;
+
+ public RangePartition(int id, String lowerBoundary, String upperBoundary) {
+ this.partitionId = id;
+ this.lowerBoundary = lowerBoundary;
+ this.upperBoundary = upperBoundary;
+ }
+
+ public String getLowerBoundary() {
+ return lowerBoundary;
+ }
+
+ public String getUpperBoundary() {
+ return upperBoundary;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f4d081e8/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index fae9960..ce5cbe4 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -27,11 +27,13 @@ import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.BucketingInfo;
+import org.apache.carbondata.core.metadata.schema.PartitionInfo;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonImplicitDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
+import org.apache.carbondata.core.stats.PartitionStatistic;
/**
* Mapping class for Carbon actual table
@@ -92,6 +94,15 @@ public class CarbonTable implements Serializable {
private Map<String, BucketingInfo> tableBucketMap;
/**
+ * table partition info
+ */
+ private Map<String, PartitionInfo> tablePartitionMap;
+
+ /**
+ * statistic information of partition table
+ */
+ private PartitionStatistic partitionStatistic;
+ /**
* tableUniqueName
*/
private String tableUniqueName;
@@ -131,6 +142,7 @@ public class CarbonTable implements Serializable {
this.tableImplicitDimensionsMap = new HashMap<String, List<CarbonDimension>>();
this.tableMeasuresMap = new HashMap<String, List<CarbonMeasure>>();
this.tableBucketMap = new HashMap<>();
+ this.tablePartitionMap = new HashMap<>();
this.aggregateTablesName = new ArrayList<String>();
this.createOrderColumn = new HashMap<String, List<CarbonColumn>>();
this.tablePrimitiveDimensionsMap = new HashMap<String, List<CarbonDimension>>();
@@ -566,6 +578,14 @@ public class CarbonTable implements Serializable {
return tableBucketMap.get(tableName);
}
+ public PartitionInfo getPartitionInfo(String tableName) {
+ return tablePartitionMap.get(tableName);
+ }
+
+ public PartitionStatistic getPartitionStatistic() {
+ return partitionStatistic;
+ }
+
/**
* @return absolute table identifier
*/
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f4d081e8/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
index 179f4d4..ad2730b 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
@@ -23,6 +23,7 @@ import java.util.Map;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.metadata.schema.BucketingInfo;
+import org.apache.carbondata.core.metadata.schema.PartitionInfo;
import org.apache.carbondata.core.metadata.schema.SchemaEvolution;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
@@ -66,6 +67,11 @@ public class TableSchema implements Serializable {
*/
private BucketingInfo bucketingInfo;
+ /**
+ * Information about partition type, partition column, numbers
+ */
+ private PartitionInfo partitionInfo;
+
public TableSchema() {
this.listOfColumns = new ArrayList<ColumnSchema>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
}
@@ -184,4 +190,12 @@ public class TableSchema implements Serializable {
public void setBucketingInfo(BucketingInfo bucketingInfo) {
this.bucketingInfo = bucketingInfo;
}
+
+ public PartitionInfo getPartitionInfo() {
+ return partitionInfo;
+ }
+
+ public void setpartitionInfo(PartitionInfo partitionInfo) {
+ this.partitionInfo = partitionInfo;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f4d081e8/core/src/main/java/org/apache/carbondata/core/stats/PartitionStatistic.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/stats/PartitionStatistic.java b/core/src/main/java/org/apache/carbondata/core/stats/PartitionStatistic.java
new file mode 100644
index 0000000..44f62b1
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/stats/PartitionStatistic.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.stats;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.carbondata.core.metadata.schema.partition.AbstractPartition;
+
+public class PartitionStatistic implements Serializable {
+
+ /**
+ * total count of partitions
+ */
+ private int numberOfPartitions;
+
+ /**
+ * partition id, increase only
+ */
+ private int partitionIndex;
+
+ private Map<Integer, AbstractPartition> partitionMap;
+
+ public PartitionStatistic() {
+ this.partitionIndex = 0;
+ this.numberOfPartitions = 0;
+ this.partitionMap = new HashMap<>();
+ }
+
+ public void addNewPartition(int id, AbstractPartition partition) {
+ partitionMap.put(id, partition);
+ partitionIndex ++;
+ numberOfPartitions ++;
+ }
+
+ public void deletePartition(int id) {
+ partitionMap.remove(id);
+ numberOfPartitions --;
+ }
+
+ public int getNumberOfPartitions() {
+ return numberOfPartitions;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f4d081e8/format/src/main/thrift/schema.thrift
----------------------------------------------------------------------
diff --git a/format/src/main/thrift/schema.thrift b/format/src/main/thrift/schema.thrift
index fba70ea..695333c 100644
--- a/format/src/main/thrift/schema.thrift
+++ b/format/src/main/thrift/schema.thrift
@@ -49,6 +49,12 @@ enum Encoding{
DIRECT_DICTIONARY = 5; // Identifies that a column is direct dictionary encoded
}
+enum PartitionType{
+ RANGE = 0;
+ RANGE_INTERVAL = 1;
+ LIST = 2;
+ HASH = 3;
+}
/**
* Description of a Column for both dimension and measure
@@ -121,6 +127,17 @@ struct SchemaEvolution{
}
/**
+ * Partition information of table
+ */
+struct PartitionInfo{
+ 1: required list<ColumnSchema> partition_columns;
+ 2: required PartitionType partition_type;
+ 3: optional i32 hash_number; // number of partitions defined in hash partition table
+ 4: optional list<list<string>> list_info; // value list of list partition table
+ 5: optional list<string> range_info; // range value list of range partition table
+}
+
+/**
* Bucketing information of fields on table
*/
struct BucketingInfo{
@@ -137,6 +154,7 @@ struct TableSchema{
3: required SchemaEvolution schema_evolution; // History of schema evolution of this table
4: optional map<string,string> tableProperties; // Table properties configured by the user
5: optional BucketingInfo bucketingInfo; // Bucketing information
+ 6: optional PartitionInfo partitionInfo; // Partition information
}
struct TableInfo{
[09/19] carbondata git commit: fix sort_columns issue
Posted by ra...@apache.org.
fix sort_columns issue
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/f5c6f288
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/f5c6f288
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/f5c6f288
Branch: refs/heads/master
Commit: f5c6f2880d4db863494ad9491c24438010c5340d
Parents: 81149f6
Author: QiangCai <qi...@qq.com>
Authored: Sat Apr 8 22:10:17 2017 +0530
Committer: jackylk <ja...@huawei.com>
Committed: Sun May 14 20:40:12 2017 +0800
----------------------------------------------------------------------
.../executer/RowLevelFilterExecuterImpl.java | 3 ++-
.../RowLevelRangeFilterResolverImpl.java | 5 +++-
.../apache/carbondata/core/util/ByteUtil.java | 28 +++++++++++---------
.../carbondata/core/util/DataTypeUtil.java | 2 +-
.../testsuite/sortcolumns/TestSortColumns.scala | 11 ++++++--
.../CarbonRowDataWriterProcessorStepImpl.java | 4 +--
6 files changed, 33 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f5c6f288/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
index 8e7c95f..38204e9 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
@@ -254,7 +254,8 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
memberBytes = null;
}
record[dimColumnEvaluatorInfo.getRowIndex()] = DataTypeUtil
- .getDataBasedOnDataType(memberBytes, dimColumnEvaluatorInfo.getDimension());
+ .getDataBasedOnDataTypeForNoDictionaryColumn(memberBytes,
+ dimColumnEvaluatorInfo.getDimension().getDataType());
} else {
continue;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f5c6f288/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
index 2a1af65..6d4e528 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
@@ -43,6 +43,7 @@ import org.apache.carbondata.core.scan.filter.intf.FilterExecuterType;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
import org.apache.carbondata.core.util.ByteUtil;
+import org.apache.carbondata.core.util.DataTypeUtil;
public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverImpl {
@@ -147,7 +148,9 @@ public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverIm
filterValuesList.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL.getBytes());
continue;
}
- filterValuesList.add(result.getString().getBytes());
+ filterValuesList.add(DataTypeUtil
+ .getBytesBasedOnDataTypeForNoDictionaryColumn(result.getString(),
+ result.getDataType()));
} catch (FilterIllegalMemberException e) {
// Any invalid member while evaluation shall be ignored, system will log the
// error only once since all rows the evaluation happens so inorder to avoid
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f5c6f288/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
index f3f0fcd..70ddec9 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
@@ -404,6 +404,7 @@ public final class ByteUtil {
* @return
*/
public static byte[] toBytes(short val) {
+ val = (short)(val ^ Short.MIN_VALUE);
byte[] b = new byte[SIZEOF_SHORT];
b[1] = (byte) val;
val >>= 8;
@@ -423,20 +424,21 @@ public final class ByteUtil {
if (length != SIZEOF_SHORT || offset + length > bytes.length) {
throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_SHORT);
}
+ short n = 0;
if (CarbonUnsafe.unsafe != null) {
if (CarbonUnsafe.ISLITTLEENDIAN) {
- return Short.reverseBytes(
+ n = Short.reverseBytes(
CarbonUnsafe.unsafe.getShort(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET));
} else {
- return CarbonUnsafe.unsafe.getShort(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
+ n = CarbonUnsafe.unsafe.getShort(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
}
} else {
- short n = 0;
+
n ^= bytes[offset] & 0xFF;
n <<= 8;
n ^= bytes[offset + 1] & 0xFF;
- return n;
}
+ return (short)(n ^ Short.MIN_VALUE);
}
/**
@@ -446,6 +448,7 @@ public final class ByteUtil {
* @return
*/
public static byte[] toBytes(int val) {
+ val = val ^ Integer.MIN_VALUE;
byte[] b = new byte[4];
for (int i = 3; i > 0; i--) {
b[i] = (byte) val;
@@ -467,21 +470,21 @@ public final class ByteUtil {
if (length != SIZEOF_INT || offset + length > bytes.length) {
throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_INT);
}
+ int n = 0;
if (CarbonUnsafe.unsafe != null) {
if (CarbonUnsafe.ISLITTLEENDIAN) {
- return Integer.reverseBytes(
+ n = Integer.reverseBytes(
CarbonUnsafe.unsafe.getInt(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET));
} else {
- return CarbonUnsafe.unsafe.getInt(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
+ n = CarbonUnsafe.unsafe.getInt(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
}
} else {
- int n = 0;
for (int i = offset; i < (offset + length); i++) {
n <<= 8;
n ^= bytes[i] & 0xFF;
}
- return n;
}
+ return n ^ Integer.MIN_VALUE;
}
/**
@@ -513,6 +516,7 @@ public final class ByteUtil {
* @return
*/
public static byte[] toBytes(long val) {
+ val = val ^ Long.MIN_VALUE;
byte[] b = new byte[8];
for (int i = 7; i > 0; i--) {
b[i] = (byte) val;
@@ -529,21 +533,21 @@ public final class ByteUtil {
if (length != SIZEOF_LONG || offset + length > bytes.length) {
throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_LONG);
}
+ long l = 0;
if (CarbonUnsafe.unsafe != null) {
if (CarbonUnsafe.ISLITTLEENDIAN) {
- return Long.reverseBytes(
+ l = Long.reverseBytes(
CarbonUnsafe.unsafe.getLong(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET));
} else {
- return CarbonUnsafe.unsafe.getLong(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
+ l = CarbonUnsafe.unsafe.getLong(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
}
} else {
- long l = 0;
for (int i = offset; i < offset + length; i++) {
l <<= 8;
l ^= bytes[i] & 0xFF;
}
- return l;
}
+ return l ^ Long.MIN_VALUE;
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f5c6f288/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
index 2e16b0f..80c9e72 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -324,7 +324,7 @@ public final class DataTypeUtil {
}
public static byte[] getBytesBasedOnDataTypeForNoDictionaryColumn(String dimensionValue,
- DataType actualDataType) throws Throwable {
+ DataType actualDataType) {
switch (actualDataType) {
case STRING:
return ByteUtil.toBytes(dimensionValue);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f5c6f288/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
index 88b0421..afdca21 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
@@ -220,13 +220,20 @@ class TestSortColumns extends QueryTest with BeforeAndAfterAll {
sql("CREATE TABLE unsortedtable_offheap_inmemory (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_offheap_inmemory OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno = 11"), sql("select * from origintable1 where empno = 11"))
+ checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno < 15 order by empno"), sql("select * from origintable1 where empno < 15 order by empno"))
+ checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno <= 15 order by empno"), sql("select * from origintable1 where empno <= 15 order by empno"))
+ checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno > 15 order by empno"), sql("select * from origintable1 where empno > 15 order by empno"))
+ checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno >= 15 order by empno"), sql("select * from origintable1 where empno >= 15 order by empno"))
+ checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno <> 15 order by empno"), sql("select * from origintable1 where empno <> 15 order by empno"))
+ checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno in (15, 16, 17) order by empno"), sql("select * from origintable1 where empno in (15, 16, 17) order by empno"))
+ checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno is null"), sql("select * from origintable1 where empno is null order by empno"))
+ checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno is not null"), sql("select * from origintable1 where empno is not null order by empno"))
checkAnswer(sql("select * from unsortedtable_offheap_inmemory order by empno"), sql("select * from origintable1 order by empno"))
} finally {
defaultLoadingProperties
}
}
-
-
+
override def afterAll = {
dropTable
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f5c6f288/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java
index 929a09e..c50f335 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java
@@ -156,9 +156,7 @@ public class CarbonRowDataWriterProcessorStepImpl extends AbstractDataLoadProces
private void doExecute(Iterator<CarbonRowBatch> iterator, int partitionId, int iteratorIndex) {
String storeLocation = getStoreLocation(tableIdentifier, String.valueOf(partitionId));
CarbonFactDataHandlerModel model = CarbonFactDataHandlerModel
- .createCarbonFactDataHandlerModel(configuration, storeLocation, partitionId, 0);
- model.getCarbonDataFileAttributes()
- .setFactTimeStamp(model.getCarbonDataFileAttributes().getFactTimeStamp() + iteratorIndex);
+ .createCarbonFactDataHandlerModel(configuration, storeLocation, partitionId, iteratorIndex);
CarbonFactHandler dataHandler = null;
boolean rowsNotExist = true;
while (iterator.hasNext()) {
[06/19] carbondata git commit: fix issue of rebase
Posted by ra...@apache.org.
fix issue of rebase
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/bf44c9f6
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/bf44c9f6
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/bf44c9f6
Branch: refs/heads/master
Commit: bf44c9f6ee4b059c384703ab12aebea4f4fed5e2
Parents: 26de62c
Author: QiangCai <qi...@qq.com>
Authored: Tue May 2 14:58:16 2017 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Sun May 14 20:40:12 2017 +0800
----------------------------------------------------------------------
.../impl/VariableLengthDimensionDataChunk.java | 3 +-
...feVariableLengthDimensionDataChunkStore.java | 32 +++++++++++++++++++-
.../apache/carbondata/core/util/ByteUtil.java | 4 +++
3 files changed, 36 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bf44c9f6/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
index 0ac5655..6c47bf5 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
@@ -18,8 +18,7 @@ package org.apache.carbondata.core.datastore.chunk.impl;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory
- .DimensionStoreType;
+import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory.DimensionStoreType;
import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bf44c9f6/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java
index bb0edea..a65d745 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java
@@ -23,6 +23,17 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
import org.apache.carbondata.core.util.ByteUtil;
+import org.apache.spark.sql.types.BooleanType;
+import org.apache.spark.sql.types.DataType;
+import org.apache.spark.sql.types.Decimal;
+import org.apache.spark.sql.types.DecimalType;
+import org.apache.spark.sql.types.DoubleType;
+import org.apache.spark.sql.types.FloatType;
+import org.apache.spark.sql.types.IntegerType;
+import org.apache.spark.sql.types.LongType;
+import org.apache.spark.sql.types.ShortType;
+import org.apache.spark.sql.types.StringType;
+
/**
* Below class is responsible to store variable length dimension data chunk in
* memory Memory occupied can be on heap or offheap using unsafe interface
@@ -137,7 +148,26 @@ public class SafeVariableLengthDimensionDataChunkStore extends SafeAbsractDimens
CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY.length, data, currentDataOffset, length)) {
vector.putNull(vectorRow);
} else {
- vector.putBytes(vectorRow, currentDataOffset, length, data);
+ DataType dt = vector.getType();
+ if (dt instanceof StringType) {
+ vector.putBytes(vectorRow, currentDataOffset, length, data);
+ } else if (dt instanceof BooleanType) {
+ vector.putBoolean(vectorRow, ByteUtil.toBoolean(data[currentDataOffset]));
+ } else if (dt instanceof ShortType) {
+ vector.putShort(vectorRow, ByteUtil.toShort(data, currentDataOffset, length));
+ } else if (dt instanceof IntegerType) {
+ vector.putInt(vectorRow, ByteUtil.toInt(data, currentDataOffset, length));
+ } else if (dt instanceof FloatType) {
+ vector.putFloat(vectorRow, ByteUtil.toFloat(data, currentDataOffset));
+ } else if (dt instanceof DoubleType) {
+ vector.putDouble(vectorRow, ByteUtil.toDouble(data, currentDataOffset));
+ } else if (dt instanceof LongType) {
+ vector.putLong(vectorRow, ByteUtil.toLong(data, currentDataOffset, length));
+ } else if (dt instanceof DecimalType) {
+ vector.putDecimal(vectorRow,
+ Decimal.apply(ByteUtil.toBigDecimal(data, currentDataOffset, length)),
+ DecimalType.MAX_PRECISION());
+ }
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bf44c9f6/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
index 70ddec9..f4ab982 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
@@ -397,6 +397,10 @@ public final class ByteUtil {
return b[0] != (byte) 0;
}
+ public static boolean toBoolean(final byte b) {
+ return b != (byte) 0;
+ }
+
/**
* short => byte[]
*
[05/19] carbondata git commit: [CARBONDATA-842] no sort datawriter
Posted by ra...@apache.org.
[CARBONDATA-842] no sort datawriter
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/48273151
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/48273151
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/48273151
Branch: refs/heads/master
Commit: 48273151baec153ab813dc9ef129daf3264bae7f
Parents: 9f94529
Author: QiangCai <qi...@qq.com>
Authored: Thu Apr 6 00:09:06 2017 +0530
Committer: jackylk <ja...@huawei.com>
Committed: Sun May 14 20:40:11 2017 +0800
----------------------------------------------------------------------
.../processing/merger/CarbonDataMergerUtil.java | 2 +-
.../processing/model/CarbonLoadModel.java | 6 +-
.../newflow/DataLoadProcessBuilder.java | 38 ++-
.../CarbonRowDataWriterProcessorStepImpl.java | 321 +++++++++++++++++++
.../newflow/steps/NoSortProcessorStepImpl.java | 153 ---------
.../store/CarbonDataFileAttributes.java | 10 +-
.../store/CarbonFactDataHandlerModel.java | 2 +-
.../store/writer/AbstractFactDataWriter.java | 4 +-
8 files changed, 361 insertions(+), 175 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/48273151/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
index 65ec4fc..b9ea447 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
@@ -170,7 +170,7 @@ public final class CarbonDataMergerUtil {
boolean updateLockStatus = false;
boolean tableLockStatus = false;
- String timestamp = carbonLoadModel.getFactTimeStamp();
+ String timestamp = "" + carbonLoadModel.getFactTimeStamp();
List<String> updatedDeltaFilesList =
new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/48273151/processing/src/main/java/org/apache/carbondata/processing/model/CarbonLoadModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/model/CarbonLoadModel.java b/processing/src/main/java/org/apache/carbondata/processing/model/CarbonLoadModel.java
index d8f84bf..f63cb03 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/model/CarbonLoadModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/model/CarbonLoadModel.java
@@ -82,7 +82,7 @@ public class CarbonLoadModel implements Serializable {
/**
* new load start time
*/
- private String factTimeStamp;
+ private long factTimeStamp;
/**
* load Id
*/
@@ -588,7 +588,7 @@ public class CarbonLoadModel implements Serializable {
/**
* @return
*/
- public String getFactTimeStamp() {
+ public long getFactTimeStamp() {
return factTimeStamp;
}
@@ -596,7 +596,7 @@ public class CarbonLoadModel implements Serializable {
* @param factTimeStamp
*/
public void setFactTimeStamp(long factTimeStamp) {
- this.factTimeStamp = factTimeStamp + "";
+ this.factTimeStamp = factTimeStamp;
}
public String[] getDelimiters() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/48273151/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
index 11e5db1..de469f9 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
@@ -35,12 +35,12 @@ import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.processing.model.CarbonLoadModel;
import org.apache.carbondata.processing.newflow.constants.DataLoadProcessorConstants;
+import org.apache.carbondata.processing.newflow.steps.CarbonRowDataWriterProcessorStepImpl;
import org.apache.carbondata.processing.newflow.steps.DataConverterProcessorStepImpl;
import org.apache.carbondata.processing.newflow.steps.DataConverterProcessorWithBucketingStepImpl;
import org.apache.carbondata.processing.newflow.steps.DataWriterBatchProcessorStepImpl;
import org.apache.carbondata.processing.newflow.steps.DataWriterProcessorStepImpl;
import org.apache.carbondata.processing.newflow.steps.InputProcessorStepImpl;
-import org.apache.carbondata.processing.newflow.steps.NoSortProcessorStepImpl;
import org.apache.carbondata.processing.newflow.steps.SortProcessorStepImpl;
import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
@@ -59,7 +59,9 @@ public final class DataLoadProcessBuilder {
CarbonCommonConstants.LOAD_USE_BATCH_SORT_DEFAULT));
CarbonDataLoadConfiguration configuration =
createConfiguration(loadModel, storeLocation);
- if (configuration.getBucketingInfo() != null) {
+ if (!configuration.isSortTable()) {
+ return buildInternalForNoSort(inputIterators, configuration);
+ } else if (configuration.getBucketingInfo() != null) {
return buildInternalForBucketing(inputIterators, configuration);
} else if (batchSort) {
return buildInternalForBatchSort(inputIterators, configuration);
@@ -77,16 +79,30 @@ public final class DataLoadProcessBuilder {
// data types and configurations.
AbstractDataLoadProcessorStep converterProcessorStep =
new DataConverterProcessorStepImpl(configuration, inputProcessorStep);
- // 3. Sorts the data by SortColumn or not
- AbstractDataLoadProcessorStep sortProcessorStep = configuration.isSortTable() ?
- new SortProcessorStepImpl(configuration, converterProcessorStep) :
- new NoSortProcessorStepImpl(configuration, converterProcessorStep);
+ // 3. Sorts the data by SortColumn
+ AbstractDataLoadProcessorStep sortProcessorStep =
+ new SortProcessorStepImpl(configuration, converterProcessorStep);
// 4. Writes the sorted data in carbondata format.
AbstractDataLoadProcessorStep writerProcessorStep =
new DataWriterProcessorStepImpl(configuration, sortProcessorStep);
return writerProcessorStep;
}
+ private AbstractDataLoadProcessorStep buildInternalForNoSort(CarbonIterator[] inputIterators,
+ CarbonDataLoadConfiguration configuration) {
+ // 1. Reads the data input iterators and parses the data.
+ AbstractDataLoadProcessorStep inputProcessorStep =
+ new InputProcessorStepImpl(configuration, inputIterators);
+ // 2. Converts the data like dictionary or non dictionary or complex objects depends on
+ // data types and configurations.
+ AbstractDataLoadProcessorStep converterProcessorStep =
+ new DataConverterProcessorStepImpl(configuration, inputProcessorStep);
+ // 3. Writes the sorted data in carbondata format.
+ AbstractDataLoadProcessorStep writerProcessorStep =
+ new CarbonRowDataWriterProcessorStepImpl(configuration, converterProcessorStep);
+ return writerProcessorStep;
+ }
+
private AbstractDataLoadProcessorStep buildInternalForBatchSort(CarbonIterator[] inputIterators,
CarbonDataLoadConfiguration configuration) {
// 1. Reads the data input iterators and parses the data.
@@ -97,9 +113,8 @@ public final class DataLoadProcessBuilder {
AbstractDataLoadProcessorStep converterProcessorStep =
new DataConverterProcessorStepImpl(configuration, inputProcessorStep);
// 3. Sorts the data by SortColumn or not
- AbstractDataLoadProcessorStep sortProcessorStep = configuration.isSortTable() ?
- new SortProcessorStepImpl(configuration, converterProcessorStep) :
- new NoSortProcessorStepImpl(configuration, converterProcessorStep);
+ AbstractDataLoadProcessorStep sortProcessorStep =
+ new SortProcessorStepImpl(configuration, converterProcessorStep);
// 4. Writes the sorted data in carbondata format.
AbstractDataLoadProcessorStep writerProcessorStep =
new DataWriterBatchProcessorStepImpl(configuration, sortProcessorStep);
@@ -116,9 +131,8 @@ public final class DataLoadProcessBuilder {
AbstractDataLoadProcessorStep converterProcessorStep =
new DataConverterProcessorWithBucketingStepImpl(configuration, inputProcessorStep);
// 3. Sorts the data by SortColumn or not
- AbstractDataLoadProcessorStep sortProcessorStep = configuration.isSortTable() ?
- new SortProcessorStepImpl(configuration, converterProcessorStep) :
- new NoSortProcessorStepImpl(configuration, converterProcessorStep);
+ AbstractDataLoadProcessorStep sortProcessorStep =
+ new SortProcessorStepImpl(configuration, converterProcessorStep);
// 4. Writes the sorted data in carbondata format.
AbstractDataLoadProcessorStep writerProcessorStep =
new DataWriterProcessorStepImpl(configuration, sortProcessorStep);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/48273151/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java
new file mode 100644
index 0000000..929a09e
--- /dev/null
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java
@@ -0,0 +1,321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.processing.newflow.steps;
+
+import java.io.File;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.util.Iterator;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.keygenerator.KeyGenException;
+import org.apache.carbondata.core.keygenerator.KeyGenerator;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
+import org.apache.carbondata.core.util.DataTypeUtil;
+import org.apache.carbondata.processing.newflow.AbstractDataLoadProcessorStep;
+import org.apache.carbondata.processing.newflow.CarbonDataLoadConfiguration;
+import org.apache.carbondata.processing.newflow.DataField;
+import org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException;
+import org.apache.carbondata.processing.newflow.row.CarbonRow;
+import org.apache.carbondata.processing.newflow.row.CarbonRowBatch;
+import org.apache.carbondata.processing.store.CarbonFactDataHandlerModel;
+import org.apache.carbondata.processing.store.CarbonFactHandler;
+import org.apache.carbondata.processing.store.CarbonFactHandlerFactory;
+import org.apache.carbondata.processing.store.writer.exception.CarbonDataWriterException;
+import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
+
+/**
+ * It reads data from sorted files which are generated in previous sort step.
+ * And it writes data to carbondata file. It also generates mdk key while writing to carbondata file
+ */
+public class CarbonRowDataWriterProcessorStepImpl extends AbstractDataLoadProcessorStep {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CarbonRowDataWriterProcessorStepImpl.class.getName());
+
+ private SegmentProperties segmentProperties;
+
+ private KeyGenerator keyGenerator;
+
+ private int dimensionWithComplexCount;
+
+ private int noDictWithComplextCount;
+
+ private boolean[] isNoDictionaryDimensionColumn;
+
+ private char[] aggType;
+
+ private int dimensionCount;
+
+ private int measureCount;
+
+ private long[] readCounter;
+
+ private long[] writeCounter;
+
+ private int outputLength;
+
+ private CarbonTableIdentifier tableIdentifier;
+
+ private String tableName;
+
+ public CarbonRowDataWriterProcessorStepImpl(CarbonDataLoadConfiguration configuration,
+ AbstractDataLoadProcessorStep child) {
+ super(configuration, child);
+ }
+
+ @Override public DataField[] getOutput() {
+ return child.getOutput();
+ }
+
+ @Override public void initialize() throws IOException {
+ child.initialize();
+ }
+
+ private String getStoreLocation(CarbonTableIdentifier tableIdentifier, String partitionId) {
+ String storeLocation = CarbonDataProcessorUtil
+ .getLocalDataFolderLocation(tableIdentifier.getDatabaseName(),
+ tableIdentifier.getTableName(), String.valueOf(configuration.getTaskNo()), partitionId,
+ configuration.getSegmentId() + "", false);
+ new File(storeLocation).mkdirs();
+ return storeLocation;
+ }
+
+ @Override public Iterator<CarbonRowBatch>[] execute() throws CarbonDataLoadingException {
+ final Iterator<CarbonRowBatch>[] iterators = child.execute();
+ tableIdentifier = configuration.getTableIdentifier().getCarbonTableIdentifier();
+ tableName = tableIdentifier.getTableName();
+ try {
+ readCounter = new long[iterators.length];
+ writeCounter = new long[iterators.length];
+ dimensionWithComplexCount = configuration.getDimensionCount();
+ noDictWithComplextCount =
+ configuration.getNoDictionaryCount() + configuration.getComplexDimensionCount();
+ dimensionCount = configuration.getDimensionCount() - noDictWithComplextCount;
+ isNoDictionaryDimensionColumn =
+ CarbonDataProcessorUtil.getNoDictionaryMapping(configuration.getDataFields());
+ aggType = CarbonDataProcessorUtil
+ .getAggType(configuration.getMeasureCount(), configuration.getMeasureFields());
+
+ CarbonFactDataHandlerModel dataHandlerModel = CarbonFactDataHandlerModel
+ .createCarbonFactDataHandlerModel(configuration,
+ getStoreLocation(tableIdentifier, String.valueOf(0)), 0, 0);
+ measureCount = dataHandlerModel.getMeasureCount();
+ outputLength = measureCount + (this.noDictWithComplextCount > 0 ? 1 : 0) + 1;
+ segmentProperties = dataHandlerModel.getSegmentProperties();
+ keyGenerator = segmentProperties.getDimensionKeyGenerator();
+
+ CarbonTimeStatisticsFactory.getLoadStatisticsInstance()
+ .recordDictionaryValue2MdkAdd2FileTime(configuration.getPartitionId(),
+ System.currentTimeMillis());
+
+ if (iterators.length == 1) {
+ doExecute(iterators[0], 0, 0);
+ } else {
+ ExecutorService executorService = Executors.newFixedThreadPool(iterators.length);
+ Future[] futures = new Future[iterators.length];
+ for (int i = 0; i < iterators.length; i++) {
+ futures[i] = executorService.submit(new DataWriterRunnable(iterators[i], i));
+ }
+ for (Future future : futures) {
+ future.get();
+ }
+ }
+ } catch (CarbonDataWriterException e) {
+ LOGGER.error(e, "Failed for table: " + tableName + " in DataWriterProcessorStepImpl");
+ throw new CarbonDataLoadingException(
+ "Error while initializing data handler : " + e.getMessage());
+ } catch (Exception e) {
+ LOGGER.error(e, "Failed for table: " + tableName + " in DataWriterProcessorStepImpl");
+ throw new CarbonDataLoadingException("There is an unexpected error: " + e.getMessage(), e);
+ }
+ return null;
+ }
+
+ private void doExecute(Iterator<CarbonRowBatch> iterator, int partitionId, int iteratorIndex) {
+ String storeLocation = getStoreLocation(tableIdentifier, String.valueOf(partitionId));
+ CarbonFactDataHandlerModel model = CarbonFactDataHandlerModel
+ .createCarbonFactDataHandlerModel(configuration, storeLocation, partitionId, 0);
+ model.getCarbonDataFileAttributes()
+ .setFactTimeStamp(model.getCarbonDataFileAttributes().getFactTimeStamp() + iteratorIndex);
+ CarbonFactHandler dataHandler = null;
+ boolean rowsNotExist = true;
+ while (iterator.hasNext()) {
+ if (rowsNotExist) {
+ rowsNotExist = false;
+ dataHandler = CarbonFactHandlerFactory
+ .createCarbonFactHandler(model, CarbonFactHandlerFactory.FactHandlerType.COLUMNAR);
+ dataHandler.initialise();
+ }
+ processBatch(iterator.next(), dataHandler, iteratorIndex);
+ }
+ if (!rowsNotExist) {
+ finish(dataHandler, iteratorIndex);
+ }
+ }
+
+ @Override protected String getStepName() {
+ return "Data Writer";
+ }
+
+ private void finish(CarbonFactHandler dataHandler, int iteratorIndex) {
+ try {
+ dataHandler.finish();
+ } catch (Exception e) {
+ LOGGER.error(e, "Failed for table: " + tableName + " in finishing data handler");
+ }
+ LOGGER.info("Record Processed For table: " + tableName);
+ String logMessage =
+ "Finished Carbon DataWriterProcessorStepImpl: Read: " + readCounter[iteratorIndex]
+ + ": Write: " + readCounter[iteratorIndex];
+ LOGGER.info(logMessage);
+ CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordTotalRecords(rowCounter.get());
+ processingComplete(dataHandler);
+ CarbonTimeStatisticsFactory.getLoadStatisticsInstance()
+ .recordDictionaryValue2MdkAdd2FileTime(configuration.getPartitionId(),
+ System.currentTimeMillis());
+ CarbonTimeStatisticsFactory.getLoadStatisticsInstance()
+ .recordMdkGenerateTotalTime(configuration.getPartitionId(), System.currentTimeMillis());
+ }
+
+ private void processingComplete(CarbonFactHandler dataHandler) throws CarbonDataLoadingException {
+ if (null != dataHandler) {
+ try {
+ dataHandler.closeHandler();
+ } catch (CarbonDataWriterException e) {
+ LOGGER.error(e, e.getMessage());
+ throw new CarbonDataLoadingException(e.getMessage());
+ } catch (Exception e) {
+ LOGGER.error(e, e.getMessage());
+ throw new CarbonDataLoadingException("There is an unexpected error: " + e.getMessage());
+ }
+ }
+ }
+
+ /**
+ * convert input CarbonRow to output CarbonRow
+ * e.g. There is a table as following,
+ * the number of dictionary dimensions is a,
+ * the number of no-dictionary dimensions is b,
+ * the number of complex dimensions is c,
+ * the number of measures is d.
+ * input CarbonRow format: the length of Object[] data is a+b+c+d, the number of all columns.
+ * ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ * | Part | Object item | describe |
+ * ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ * | Object[0 ~ a+b-1] | Integer, byte[], Integer, ... | dict + no dict dimensions|
+ * ----------------------------------------------------------------------------------------
+ * | Object[a+b ~ a+b+c-1] | byte[], byte[], ... | complex dimensions |
+ * ----------------------------------------------------------------------------------------
+ * | Object[a+b+c ~ a+b+c+d-1]| int, byte[], ... | measures |
+ * ----------------------------------------------------------------------------------------
+ * output CarbonRow format: the length of object[] data is d + (b+c>0?1:0) + 1.
+ * ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ * | Part | Object item | describe |
+ * ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ * | Object[0 ~ d-1] | int, byte[], ... | measures |
+ * ----------------------------------------------------------------------------------------
+ * | Object[d] | byte[b+c][] | no dict + complex dim |
+ * ----------------------------------------------------------------------------------------
+ * | Object[d+1] | byte[] | mdkey |
+ * ----------------------------------------------------------------------------------------
+ *
+ * @param row
+ * @return
+ */
+ private Object[] convertRow(CarbonRow row) throws KeyGenException {
+ int dictIndex = 0;
+ int nonDicIndex = 0;
+ int[] dim = new int[this.dimensionCount];
+ byte[][] nonDicArray = new byte[this.noDictWithComplextCount][];
+ // read dimension values
+ int dimCount = 0;
+ for (; dimCount < isNoDictionaryDimensionColumn.length; dimCount++) {
+ if (isNoDictionaryDimensionColumn[dimCount]) {
+ nonDicArray[nonDicIndex++] = (byte[]) row.getObject(dimCount);
+ } else {
+ dim[dictIndex++] = (int) row.getObject(dimCount);
+ }
+ }
+
+ for (; dimCount < this.dimensionWithComplexCount; dimCount++) {
+ nonDicArray[nonDicIndex++] = (byte[]) row.getObject(dimCount);
+ }
+
+ int l = 0;
+ Object[] outputRow = new Object[outputLength];
+ for (; l < this.measureCount; l++) {
+ Object value = row.getObject(l + this.dimensionWithComplexCount);
+ if (null != value) {
+ if (aggType[l] == CarbonCommonConstants.BIG_DECIMAL_MEASURE) {
+ BigDecimal val = (BigDecimal) value;
+ outputRow[l] = DataTypeUtil.bigDecimalToByte(val);
+ } else {
+ outputRow[l] = value;
+ }
+ } else {
+ outputRow[l] = null;
+ }
+ }
+
+ if (this.noDictWithComplextCount > 0) {
+ outputRow[l++] = nonDicArray;
+ }
+ outputRow[l] = keyGenerator.generateKey(dim);
+ return outputRow;
+ }
+
+ private void processBatch(CarbonRowBatch batch, CarbonFactHandler dataHandler, int iteratorIndex)
+ throws CarbonDataLoadingException {
+ try {
+ while (batch.hasNext()) {
+ dataHandler.addDataToStore(convertRow(batch.next()));
+ readCounter[iteratorIndex]++;
+ }
+ writeCounter[iteratorIndex] += batch.getSize();
+ } catch (Exception e) {
+ throw new CarbonDataLoadingException("unable to generate the mdkey", e);
+ }
+ rowCounter.getAndAdd(batch.getSize());
+ }
+
+ @Override protected CarbonRow processRow(CarbonRow row) {
+ return null;
+ }
+
+ class DataWriterRunnable implements Runnable {
+
+ private Iterator<CarbonRowBatch> iterator;
+ private int iteratorIndex = 0;
+
+ DataWriterRunnable(Iterator<CarbonRowBatch> iterator, int iteratorIndex) {
+ this.iterator = iterator;
+ this.iteratorIndex = iteratorIndex;
+ }
+
+ @Override public void run() {
+ doExecute(this.iterator, 0, iteratorIndex);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/48273151/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/NoSortProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/NoSortProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/NoSortProcessorStepImpl.java
deleted file mode 100644
index bde89ed..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/NoSortProcessorStepImpl.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.processing.newflow.steps;
-
-import java.io.IOException;
-import java.math.BigDecimal;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.util.DataTypeUtil;
-import org.apache.carbondata.processing.newflow.AbstractDataLoadProcessorStep;
-import org.apache.carbondata.processing.newflow.CarbonDataLoadConfiguration;
-import org.apache.carbondata.processing.newflow.DataField;
-import org.apache.carbondata.processing.newflow.row.CarbonRow;
-import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
-import org.apache.carbondata.processing.util.NonDictionaryUtil;
-
-/**
- * if the table doesn't have sort_columns, just convert row format.
- */
-public class NoSortProcessorStepImpl extends AbstractDataLoadProcessorStep {
-
- private int dimensionCount;
-
- private int dimensionWithComplexCount;
-
- private int noDictCount;
-
- private int measureCount;
-
- private boolean[] isNoDictionaryDimensionColumn;
-
- private char[] aggType;
-
- public NoSortProcessorStepImpl(CarbonDataLoadConfiguration configuration,
- AbstractDataLoadProcessorStep child) {
- super(configuration, child);
- this.dimensionWithComplexCount = configuration.getDimensionCount();
- this.noDictCount =
- configuration.getNoDictionaryCount() + configuration.getComplexDimensionCount();
- this.dimensionCount = configuration.getDimensionCount() - this.noDictCount;
- this.measureCount = configuration.getMeasureCount();
- this.isNoDictionaryDimensionColumn =
- CarbonDataProcessorUtil.getNoDictionaryMapping(configuration.getDataFields());
- this.aggType = CarbonDataProcessorUtil
- .getAggType(configuration.getMeasureCount(), configuration.getMeasureFields());
- }
-
- @Override public DataField[] getOutput() {
- return child.getOutput();
- }
-
- @Override public void initialize() throws IOException {
- child.initialize();
- }
-
- /**
- * convert input CarbonRow to output CarbonRow
- * e.g. There is a table as following,
- * the number of dictionary dimensions is a,
- * the number of no-dictionary dimensions is b,
- * the number of complex dimensions is c,
- * the number of measures is d.
- * input CarbonRow format: the length of Object[] data is a+b+c+d, the number of all columns.
- * ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
- * | Part | Object item | describe |
- * ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
- * | Object[0 ~ a+b-1] | Integer, byte[], Integer, ... | dict + no dict dimensions|
- * ----------------------------------------------------------------------------------------
- * | Object[a+b ~ a+b+c-1] | byte[], byte[], ... | complex dimensions |
- * ----------------------------------------------------------------------------------------
- * | Object[a+b+c ~ a+b+c+d-1]| int, byte[], ... | measures |
- * ----------------------------------------------------------------------------------------
- * output CarbonRow format: the length of object[] data is 3.
- * ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
- * | Part | Object item | describe |
- * ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
- * | Object[0] | int[a] | dict dimension array |
- * ----------------------------------------------------------------------------------------
- * | Object[1] | byte[b+c][] | no dict + complex dim |
- * ----------------------------------------------------------------------------------------
- * | Object[2] | Object[d] | measures |
- * ----------------------------------------------------------------------------------------
- * @param row
- * @return
- */
- @Override protected CarbonRow processRow(CarbonRow row) {
- int dictIndex = 0;
- int nonDicIndex = 0;
- int[] dim = new int[this.dimensionCount];
- byte[][] nonDicArray = new byte[this.noDictCount][];
- Object[] measures = new Object[this.measureCount];
- // read dimension values
- int dimCount = 0;
- for (; dimCount < isNoDictionaryDimensionColumn.length; dimCount++) {
- if (isNoDictionaryDimensionColumn[dimCount]) {
- nonDicArray[nonDicIndex++] = (byte[]) row.getObject(dimCount);
- } else {
- dim[dictIndex++] = (int) row.getObject(dimCount);
- }
- }
-
- for (; dimCount < this.dimensionWithComplexCount; dimCount++) {
- nonDicArray[nonDicIndex++] = (byte[]) row.getObject(dimCount);
- }
-
- // measure values
- for (int mesCount = 0; mesCount < this.measureCount; mesCount++) {
- Object value = row.getObject(mesCount + this.dimensionWithComplexCount);
- if (null != value) {
- if (aggType[mesCount] == CarbonCommonConstants.DOUBLE_MEASURE) {
- measures[mesCount] = value;
- } else if (aggType[mesCount] == CarbonCommonConstants.BIG_INT_MEASURE) {
- measures[mesCount] = value;
- } else if (aggType[mesCount] == CarbonCommonConstants.BIG_DECIMAL_MEASURE) {
- BigDecimal val = (BigDecimal) value;
- measures[mesCount] = DataTypeUtil.bigDecimalToByte(val);
- }
- } else {
- measures[mesCount] = null;
- }
- }
- // create new row of size 3 (1 for dims , 1 for high card , 1 for measures)
- Object[] holder = new Object[3];
- NonDictionaryUtil.prepareOutObj(holder, dim, nonDicArray, measures);
- //return out row
- return new CarbonRow(holder);
- }
-
- @Override
- public void close() {
- if (!closed) {
- super.close();
- }
- }
-
- @Override protected String getStepName() {
- return "No Sort Processor";
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/48273151/processing/src/main/java/org/apache/carbondata/processing/store/CarbonDataFileAttributes.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonDataFileAttributes.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonDataFileAttributes.java
index 188c468..0b606b0 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonDataFileAttributes.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonDataFileAttributes.java
@@ -39,13 +39,13 @@ public class CarbonDataFileAttributes {
/**
* load start time
*/
- private String factTimeStamp;
+ private long factTimeStamp;
/**
* @param taskId
* @param factTimeStamp
*/
- public CarbonDataFileAttributes(int taskId, String factTimeStamp) {
+ public CarbonDataFileAttributes(int taskId, long factTimeStamp) {
this.taskId = taskId;
this.factTimeStamp = factTimeStamp;
}
@@ -57,10 +57,14 @@ public class CarbonDataFileAttributes {
return taskId;
}
+ public void setFactTimeStamp(long factTimeStamp) {
+ this.factTimeStamp = factTimeStamp;
+ }
+
/**
* @return fact time stamp which is load start time
*/
- public String getFactTimeStamp() {
+ public long getFactTimeStamp() {
return factTimeStamp;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/48273151/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
index d3b5605..61322bc 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
@@ -265,7 +265,7 @@ public class CarbonFactDataHandlerModel {
CarbonDataFileAttributes carbonDataFileAttributes =
new CarbonDataFileAttributes(Integer.parseInt(configuration.getTaskNo()),
- (String) configuration.getDataLoadProperty(DataLoadProcessorConstants.FACT_TIME_STAMP));
+ (Long) configuration.getDataLoadProperty(DataLoadProcessorConstants.FACT_TIME_STAMP));
String carbonDataDirectoryPath = getCarbonDataFolderLocation(configuration);
CarbonFactDataHandlerModel carbonFactDataHandlerModel = new CarbonFactDataHandlerModel();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/48273151/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
index cda907c..770d24c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
@@ -296,7 +296,7 @@ public abstract class AbstractFactDataWriter<T> implements CarbonFactDataWriter<
this.carbonDataFileName = carbonTablePath
.getCarbonDataFileName(fileCount, dataWriterVo.getCarbonDataFileAttributes().getTaskId(),
dataWriterVo.getBucketNumber(), dataWriterVo.getTaskExtension(),
- dataWriterVo.getCarbonDataFileAttributes().getFactTimeStamp());
+ "" + dataWriterVo.getCarbonDataFileAttributes().getFactTimeStamp());
String actualFileNameVal = carbonDataFileName + CarbonCommonConstants.FILE_INPROGRESS_STATUS;
FileData fileData = new FileData(actualFileNameVal, dataWriterVo.getStoreLocation());
dataWriterVo.getFileManager().add(fileData);
@@ -447,7 +447,7 @@ public abstract class AbstractFactDataWriter<T> implements CarbonFactDataWriter<
String fileName = dataWriterVo.getStoreLocation() + File.separator + carbonTablePath
.getCarbonIndexFileName(dataWriterVo.getCarbonDataFileAttributes().getTaskId(),
dataWriterVo.getBucketNumber(), dataWriterVo.getTaskExtension(),
- dataWriterVo.getCarbonDataFileAttributes().getFactTimeStamp());
+ "" + dataWriterVo.getCarbonDataFileAttributes().getFactTimeStamp());
CarbonIndexFileWriter writer = new CarbonIndexFileWriter();
// open file
writer.openThriftWriter(fileName);
[03/19] carbondata git commit: [CARBONDATA-782] add SORT_COLUMNS
options
Posted by ra...@apache.org.
[CARBONDATA-782] add SORT_COLUMNS options
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/9f945291
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/9f945291
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/9f945291
Branch: refs/heads/master
Commit: 9f94529112c8d955dd6ebfc26a890e784cd7b125
Parents: fb2441a
Author: QiangCai <qi...@qq.com>
Authored: Tue Apr 18 17:12:14 2017 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Sun May 14 20:39:57 2017 +0800
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 1 +
.../core/datastore/block/SegmentProperties.java | 42 +++
.../impl/VariableLengthDimensionDataChunk.java | 3 +-
.../impl/btree/BTreeDataRefNodeFinder.java | 24 +-
.../core/keygenerator/mdkey/Bits.java | 6 +-
.../ThriftWrapperSchemaConverterImpl.java | 18 +
.../core/metadata/schema/table/CarbonTable.java | 24 ++
.../schema/table/column/ColumnSchema.java | 10 +
.../impl/DictionaryBasedResultCollector.java | 12 +-
...structureBasedDictionaryResultCollector.java | 4 +-
.../scan/filter/FilterExpressionProcessor.java | 4 +-
.../carbondata/core/scan/filter/FilterUtil.java | 98 ++++--
...velRangeLessThanEqualFilterExecuterImpl.java | 2 +-
.../RowLevelRangeLessThanFiterExecuterImpl.java | 2 +-
.../visitor/NoDictionaryTypeVisitor.java | 3 +-
.../visitor/RangeNoDictionaryTypeVisitor.java | 3 +-
.../AbstractDetailQueryResultIterator.java | 4 +-
.../scan/result/vector/CarbonColumnVector.java | 7 +
.../util/AbstractDataFileFooterConverter.java | 7 +
.../apache/carbondata/core/util/ByteUtil.java | 336 ++++++++++++++++++-
.../apache/carbondata/core/util/CarbonUtil.java | 18 +
.../carbondata/core/util/DataTypeUtil.java | 70 ++++
.../impl/btree/BTreeBlockFinderTest.java | 10 +-
.../core/scan/filter/FilterUtilTest.java | 7 +-
.../carbondata/hadoop/CarbonInputFormat.java | 4 +-
.../internal/index/impl/InMemoryBTreeIndex.java | 4 +-
.../impl/DictionaryDecodeReadSupport.java | 3 +
.../hadoop/util/BlockLevelTraverser.java | 4 +-
.../hadoop/test/util/StoreCreator.java | 5 +
.../testsuite/sortcolumns/TestSortColumns.scala | 267 +++++++++++++++
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 125 +++----
.../execution/command/carbonTableSchema.scala | 66 ++--
.../apache/spark/sql/TestCarbonSqlParser.scala | 56 +---
.../readsupport/SparkRowReadSupportImpl.java | 13 +-
.../vectorreader/ColumnarVectorWrapper.java | 13 +
.../org/apache/spark/sql/TableCreator.scala | 145 ++++----
.../merger/CompactionResultSortProcessor.java | 10 +-
.../merger/RowResultMergerProcessor.java | 14 +-
.../newflow/CarbonDataLoadConfiguration.java | 24 ++
.../newflow/DataLoadProcessBuilder.java | 24 +-
.../impl/NonDictionaryFieldConverterImpl.java | 21 +-
.../sort/impl/ParallelReadMergeSorterImpl.java | 3 +-
...arallelReadMergeSorterWithBucketingImpl.java | 3 +-
.../sort/unsafe/UnsafeCarbonRowPage.java | 11 +-
.../newflow/sort/unsafe/UnsafeSortDataRows.java | 11 +-
.../unsafe/comparator/UnsafeRowComparator.java | 10 +-
.../UnsafeRowComparatorForNormalDIms.java | 8 +-
.../holder/UnsafeFinalMergePageHolder.java | 4 +-
.../unsafe/holder/UnsafeInmemoryHolder.java | 5 +-
.../holder/UnsafeSortTempFileChunkHolder.java | 2 +-
.../UnsafeSingleThreadFinalSortFilesMerger.java | 4 +-
.../newflow/steps/NoSortProcessorStepImpl.java | 153 +++++++++
.../sortdata/IntermediateFileMerger.java | 3 +-
.../sortdata/NewRowComparator.java | 12 +-
.../sortdata/NewRowComparatorForNormalDims.java | 10 +-
.../sortandgroupby/sortdata/RowComparator.java | 10 +-
.../sortdata/RowComparatorForNormalDims.java | 10 +-
.../sortandgroupby/sortdata/SortDataRows.java | 13 +-
.../sortandgroupby/sortdata/SortParameters.java | 43 +++
.../sortdata/SortTempFileChunkHolder.java | 11 +-
.../store/CarbonFactDataHandlerColumnar.java | 45 ++-
.../store/CarbonFactDataHandlerModel.java | 7 +
.../store/SingleThreadFinalSortFilesMerger.java | 8 +-
63 files changed, 1549 insertions(+), 350 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 269a75f..b4235e2 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -815,6 +815,7 @@ public final class CarbonCommonConstants {
public static final String COLUMN_GROUPS = "column_groups";
public static final String DICTIONARY_EXCLUDE = "dictionary_exclude";
public static final String DICTIONARY_INCLUDE = "dictionary_include";
+ public static final String SORT_COLUMNS = "sort_columns";
public static final String PARTITIONCLASS = "partitionclass";
public static final String PARTITIONCOUNT = "partitioncount";
public static final String COLUMN_PROPERTIES = "columnproperties";
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
index 3bc208d..38e3c02 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
@@ -55,6 +55,14 @@ public class SegmentProperties {
private KeyGenerator dimensionKeyGenerator;
/**
+ * key generator which was used to generate the mdkey for dimensions in SORT_COLUMNS
+ * if SORT_COLUMNS contains all dimensions, it is same with dimensionKeyGenerator
+ * otherwise, it is different with dimensionKeyGenerator, the number of its dimensions is less
+ * than dimensionKeyGenerator.
+ */
+ private KeyGenerator sortColumnsGenerator;
+
+ /**
* list of dimension present in the block
*/
private List<CarbonDimension> dimensions;
@@ -151,6 +159,10 @@ public class SegmentProperties {
*/
private ColumnGroupModel colGroupModel;
+ private int numberOfSortColumns = 0;
+
+ private int numberOfNoDictSortColumns = 0;
+
public SegmentProperties(List<ColumnSchema> columnsInTable, int[] columnCardinality) {
dimensions = new ArrayList<CarbonDimension>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
complexDimensions =
@@ -336,6 +348,9 @@ public class SegmentProperties {
if (CarbonUtil.hasEncoding(columnSchema.getEncodingList(), Encoding.DICTIONARY)
&& !isComplexDimensionStarted && columnSchema.getNumberOfChild() == 0) {
cardinalityIndexForNormalDimensionColumn.add(tableOrdinal);
+ if (columnSchema.isSortColumn()) {
+ this.numberOfSortColumns++;
+ }
if (columnSchema.isColumnar()) {
// if it is a columnar dimension participated in mdkey then added
// key ordinal and dimension ordinal
@@ -385,6 +400,10 @@ public class SegmentProperties {
// for no dictionary dimension
carbonDimension = new CarbonDimension(columnSchema, dimensonOrdinal++, -1, -1, -1);
numberOfNoDictionaryDimension++;
+ if (columnSchema.isSortColumn()) {
+ this.numberOfSortColumns++;
+ this.numberOfNoDictSortColumns++;
+ }
}
dimensions.add(carbonDimension);
} else {
@@ -511,6 +530,14 @@ public class SegmentProperties {
int[] bitLength = CarbonUtil.getDimensionBitLength(dimColumnsCardinality, dimensionPartitions);
// create a key generator
this.dimensionKeyGenerator = new MultiDimKeyVarLengthGenerator(bitLength);
+ if (this.getNumberOfDictSortColumns() == bitLength.length) {
+ this.sortColumnsGenerator = this.dimensionKeyGenerator;
+ } else {
+ int numberOfDictSortColumns = this.getNumberOfDictSortColumns();
+ int [] sortColumnBitLength = new int[numberOfDictSortColumns];
+ System.arraycopy(bitLength, 0, sortColumnBitLength, 0, numberOfDictSortColumns);
+ this.sortColumnsGenerator = new MultiDimKeyVarLengthGenerator(sortColumnBitLength);
+ }
this.fixedLengthKeySplitter =
new MultiDimKeyVarLengthVariableSplitGenerator(bitLength, dimensionPartitions);
// get the size of each value in file block
@@ -636,6 +663,10 @@ public class SegmentProperties {
return dimensionKeyGenerator;
}
+ public KeyGenerator getSortColumnsGenerator() {
+ return sortColumnsGenerator;
+ }
+
/**
* @return the dimensions
*/
@@ -797,4 +828,15 @@ public class SegmentProperties {
return CarbonUtil.getMeasureFromCurrentBlock(this.measures, columnId);
}
+ public int getNumberOfSortColumns() {
+ return numberOfSortColumns;
+ }
+
+ public int getNumberOfNoDictSortColumns() {
+ return numberOfNoDictSortColumns;
+ }
+
+ public int getNumberOfDictSortColumns() {
+ return this.numberOfSortColumns - this.numberOfNoDictSortColumns;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
index 6c47bf5..0ac5655 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
@@ -18,7 +18,8 @@ package org.apache.carbondata.core.datastore.chunk.impl;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory.DimensionStoreType;
+import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory
+ .DimensionStoreType;
import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeDataRefNodeFinder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeDataRefNodeFinder.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeDataRefNodeFinder.java
index 2e1709b..2f8aadf 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeDataRefNodeFinder.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeDataRefNodeFinder.java
@@ -49,18 +49,20 @@ public class BTreeDataRefNodeFinder implements DataRefNodeFinder {
private int[] eachColumnValueSize;
/**
- * this will be used during search for no dictionary column
+ * the number of no dictionary columns in SORT_COLUMNS
*/
- private int numberOfNoDictionaryColumns;
+ private int numberOfNoDictSortColumns;
- public BTreeDataRefNodeFinder(int[] eachColumnValueSize) {
- this.eachColumnValueSize = eachColumnValueSize;
+ /**
+ * the number of columns in SORT_COLUMNS
+ */
+ private int numberOfSortColumns;
- for (int i = 0; i < eachColumnValueSize.length; i++) {
- if (eachColumnValueSize[i] == -1) {
- numberOfNoDictionaryColumns++;
- }
- }
+ public BTreeDataRefNodeFinder(int[] eachColumnValueSize, int numberOfSortColumns,
+ int numberOfNoDictSortColumns) {
+ this.eachColumnValueSize = eachColumnValueSize;
+ this.numberOfNoDictSortColumns = numberOfNoDictSortColumns;
+ this.numberOfSortColumns = numberOfSortColumns;
}
/**
@@ -213,7 +215,7 @@ public class BTreeDataRefNodeFinder implements DataRefNodeFinder {
int dictionaryKeyOffset = 0;
int nonDictionaryKeyOffset = 0;
int compareResult = 0;
- int processedNoDictionaryColumn = numberOfNoDictionaryColumns;
+ int processedNoDictionaryColumn = numberOfNoDictSortColumns;
ByteBuffer firstNoDictionaryKeyBuffer = ByteBuffer.wrap(first.getNoDictionaryKeys());
ByteBuffer secondNoDictionaryKeyBuffer = ByteBuffer.wrap(second.getNoDictionaryKeys());
int actualOffset = 0;
@@ -221,7 +223,7 @@ public class BTreeDataRefNodeFinder implements DataRefNodeFinder {
int firstNoDcitionaryLength = 0;
int secondNodeDictionaryLength = 0;
- for (int i = 0; i < eachColumnValueSize.length; i++) {
+ for (int i = 0; i < numberOfSortColumns; i++) {
if (eachColumnValueSize[i] != NO_DCITIONARY_COLUMN_VALUE) {
compareResult = ByteUtil.UnsafeComparer.INSTANCE
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/Bits.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/Bits.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/Bits.java
index 7bd338d..044a77b 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/Bits.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/Bits.java
@@ -119,7 +119,8 @@ public class Bits implements Serializable {
protected long[] get(long[] keys) {
long[] words = new long[wsize];
int ll = 0;
- for (int i = lens.length - 1; i >= 0; i--) {
+ int minLength = Math.min(lens.length, keys.length);
+ for (int i = minLength - 1; i >= 0; i--) {
long val = keys[i];
@@ -155,7 +156,8 @@ public class Bits implements Serializable {
protected long[] get(int[] keys) {
long[] words = new long[wsize];
int ll = 0;
- for (int i = lens.length - 1; i >= 0; i--) {
+ int minLength = Math.min(lens.length, keys.length);
+ for (int i = minLength - 1; i >= 0; i--) {
long val = keys[i];
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java b/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
index 974cc81..3d114ec 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
@@ -17,8 +17,11 @@
package org.apache.carbondata.core.metadata.converter;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.BucketingInfo;
@@ -172,6 +175,13 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
thriftColumnSchema.setInvisible(wrapperColumnSchema.isInvisible());
thriftColumnSchema.setColumnReferenceId(wrapperColumnSchema.getColumnReferenceId());
thriftColumnSchema.setSchemaOrdinal(wrapperColumnSchema.getSchemaOrdinal());
+
+ if (wrapperColumnSchema.isSortColumn()) {
+ Map<String, String> properties = new HashMap<String, String>();
+ properties.put(CarbonCommonConstants.SORT_COLUMNS, "true");
+ thriftColumnSchema.setColumnProperties(properties);
+ }
+
return thriftColumnSchema;
}
@@ -360,6 +370,14 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
wrapperColumnSchema.setInvisible(externalColumnSchema.isInvisible());
wrapperColumnSchema.setColumnReferenceId(externalColumnSchema.getColumnReferenceId());
wrapperColumnSchema.setSchemaOrdinal(externalColumnSchema.getSchemaOrdinal());
+ wrapperColumnSchema.setSortColumn(false);
+ Map<String, String> properties = externalColumnSchema.getColumnProperties();
+ if (properties != null) {
+ String sortColumns = properties.get(CarbonCommonConstants.SORT_COLUMNS);
+ if (sortColumns != null) {
+ wrapperColumnSchema.setSortColumn(true);
+ }
+ }
return wrapperColumnSchema;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index dbc8836..fae9960 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -116,6 +116,16 @@ public class CarbonTable implements Serializable {
*/
private int blockSize;
+ /**
+ * the number of columns in SORT_COLUMNS
+ */
+ private int numberOfSortColumns;
+
+ /**
+ * the number of no dictionary columns in SORT_COLUMNS
+ */
+ private int numberOfNoDictSortColumns;
+
public CarbonTable() {
this.tableDimensionsMap = new HashMap<String, List<CarbonDimension>>();
this.tableImplicitDimensionsMap = new HashMap<String, List<CarbonDimension>>();
@@ -238,10 +248,16 @@ public class CarbonTable implements Serializable {
i = dimensionOrdinal - 1;
complexTypeOrdinal = assignComplexOrdinal(complexDimension, complexTypeOrdinal);
} else {
+ if (!columnSchema.isInvisible() && columnSchema.isSortColumn()) {
+ this.numberOfSortColumns++;
+ }
if (!columnSchema.getEncodingList().contains(Encoding.DICTIONARY)) {
CarbonDimension dimension =
new CarbonDimension(columnSchema, dimensionOrdinal++,
columnSchema.getSchemaOrdinal(), -1, -1, -1);
+ if (!columnSchema.isInvisible() && columnSchema.isSortColumn()) {
+ this.numberOfNoDictSortColumns++;
+ }
allDimensions.add(dimension);
primitiveDimensions.add(dimension);
} else if (columnSchema.getEncodingList().contains(Encoding.DICTIONARY)
@@ -639,4 +655,12 @@ public class CarbonTable implements Serializable {
}
tableMeasuresMap.put(tableName, visibleMeasures);
}
+
+ public int getNumberOfSortColumns() {
+ return numberOfSortColumns;
+ }
+
+ public int getNumberOfNoDictSortColumns() {
+ return numberOfNoDictSortColumns;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
index abe065c..6250dba 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
@@ -109,6 +109,8 @@ public class ColumnSchema implements Serializable {
*/
private boolean invisible = false;
+ private boolean isSortColumn = false;
+
/**
* @return the columnName
*/
@@ -411,4 +413,12 @@ public class ColumnSchema implements Serializable {
public void setSchemaOrdinal(int schemaOrdinal) {
this.schemaOrdinal = schemaOrdinal;
}
+
+ public boolean isSortColumn() {
+ return isSortColumn;
+ }
+
+ public void setSortColumn(boolean sortColumn) {
+ isSortColumn = sortColumn;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
index e5e4b3c..a7c93bb 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
@@ -88,7 +88,7 @@ public class DictionaryBasedResultCollector extends AbstractScannedResultCollect
List<Object[]> listBasedResult = new ArrayList<>(batchSize);
int rowCounter = 0;
int[] surrogateResult;
- String[] noDictionaryKeys;
+ byte[][] noDictionaryKeys;
byte[][] complexTypeKeyArray;
BlockletLevelDeleteDeltaDataCache deleteDeltaDataCache =
scannedResult.getDeleteDeltaDataCache();
@@ -96,7 +96,7 @@ public class DictionaryBasedResultCollector extends AbstractScannedResultCollect
Object[] row = new Object[queryDimensions.length + queryMeasures.length];
if (isDimensionExists) {
surrogateResult = scannedResult.getDictionaryKeyIntegerArray();
- noDictionaryKeys = scannedResult.getNoDictionaryKeyStringArray();
+ noDictionaryKeys = scannedResult.getNoDictionaryKeyArray();
complexTypeKeyArray = scannedResult.getComplexTypeKeyArray();
dictionaryColumnIndex = 0;
noDictionaryColumnIndex = 0;
@@ -120,7 +120,7 @@ public class DictionaryBasedResultCollector extends AbstractScannedResultCollect
}
protected void fillDimensionData(AbstractScannedResult scannedResult, int[] surrogateResult,
- String[] noDictionaryKeys, byte[][] complexTypeKeyArray,
+ byte[][] noDictionaryKeys, byte[][] complexTypeKeyArray,
Map<Integer, GenericQueryType> comlexDimensionInfoMap, Object[] row, int i) {
if (!dictionaryEncodingArray[i]) {
if (implictColumnArray[i]) {
@@ -134,9 +134,9 @@ public class DictionaryBasedResultCollector extends AbstractScannedResultCollect
DataTypeUtil.getDataBasedOnDataType(scannedResult.getBlockletId(), DataType.STRING);
}
} else {
- row[order[i]] = DataTypeUtil
- .getDataBasedOnDataType(noDictionaryKeys[noDictionaryColumnIndex++],
- queryDimensions[i].getDimension().getDataType());
+ row[order[i]] = DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(
+ noDictionaryKeys[noDictionaryColumnIndex++],
+ queryDimensions[i].getDimension().getDataType());
}
} else if (directDictionaryEncodingArray[i]) {
if (directDictionaryGenerators[i] != null) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedDictionaryResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedDictionaryResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedDictionaryResultCollector.java
index 71045ff..7c07923 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedDictionaryResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedDictionaryResultCollector.java
@@ -48,7 +48,7 @@ public class RestructureBasedDictionaryResultCollector extends DictionaryBasedRe
List<Object[]> listBasedResult = new ArrayList<>(batchSize);
int rowCounter = 0;
int[] surrogateResult;
- String[] noDictionaryKeys;
+ byte[][] noDictionaryKeys;
byte[][] complexTypeKeyArray;
BlockletLevelDeleteDeltaDataCache deleteDeltaDataCache =
scannedResult.getDeleteDeltaDataCache();
@@ -58,7 +58,7 @@ public class RestructureBasedDictionaryResultCollector extends DictionaryBasedRe
Object[] row = new Object[queryDimensions.length + queryMeasures.length];
if (isDimensionExists) {
surrogateResult = scannedResult.getDictionaryKeyIntegerArray();
- noDictionaryKeys = scannedResult.getNoDictionaryKeyStringArray();
+ noDictionaryKeys = scannedResult.getNoDictionaryKeyArray();
complexTypeKeyArray = scannedResult.getComplexTypeKeyArray();
dictionaryColumnIndex = 0;
noDictionaryColumnIndex = 0;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
index 264c0ae..b5341a0 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
@@ -116,7 +116,9 @@ public class FilterExpressionProcessor implements FilterProcessor {
+ searchEndKey.getNoDictionaryKeys());
long startTimeInMillis = System.currentTimeMillis();
DataRefNodeFinder blockFinder = new BTreeDataRefNodeFinder(
- tableSegment.getSegmentProperties().getEachDimColumnValueSize());
+ tableSegment.getSegmentProperties().getEachDimColumnValueSize(),
+ tableSegment.getSegmentProperties().getNumberOfSortColumns(),
+ tableSegment.getSegmentProperties().getNumberOfNoDictSortColumns());
DataRefNode startBlock = blockFinder.findFirstDataBlock(btreeNode, searchStartKey);
DataRefNode endBlock = blockFinder.findLastDataBlock(btreeNode, searchEndKey);
FilterExecuter filterExecuter =
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
index 7799b6a..d816c69 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
@@ -355,10 +355,23 @@ public final class FilterUtil {
* @return DimColumnFilterInfo
*/
public static DimColumnFilterInfo getNoDictionaryValKeyMemberForFilter(
- List<String> evaluateResultListFinal, boolean isIncludeFilter) {
+ List<String> evaluateResultListFinal, boolean isIncludeFilter, DataType dataType)
+ throws FilterUnsupportedException {
List<byte[]> filterValuesList = new ArrayList<byte[]>(20);
- for (String result : evaluateResultListFinal) {
- filterValuesList.add(result.getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)));
+ String result = null;
+ try {
+ int length = evaluateResultListFinal.size();
+ for (int i = 0; i < length; i++) {
+ result = evaluateResultListFinal.get(i);
+ if (CarbonCommonConstants.MEMBER_DEFAULT_VAL.equals(result)) {
+ filterValuesList.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY);
+ continue;
+ }
+ filterValuesList.add(
+ DataTypeUtil.getBytesBasedOnDataTypeForNoDictionaryColumn(result, dataType));
+ }
+ } catch (Throwable ex) {
+ throw new FilterUnsupportedException("Unsupported Filter condition: " + result, ex);
}
Comparator<byte[]> filterNoDictValueComaparator = new Comparator<byte[]>() {
@@ -879,8 +892,10 @@ public final class FilterUtil {
}
int keyOrdinalOfDimensionFromCurrentBlock = dimensionFromCurrentBlock.getKeyOrdinal();
for (DimColumnFilterInfo info : values) {
- if (startKey[keyOrdinalOfDimensionFromCurrentBlock] < info.getFilterList().get(0)) {
- startKey[keyOrdinalOfDimensionFromCurrentBlock] = info.getFilterList().get(0);
+ if (keyOrdinalOfDimensionFromCurrentBlock < startKey.length) {
+ if (startKey[keyOrdinalOfDimensionFromCurrentBlock] < info.getFilterList().get(0)) {
+ startKey[keyOrdinalOfDimensionFromCurrentBlock] = info.getFilterList().get(0);
+ }
}
}
long[] newStartKey = new long[startKey.length];
@@ -944,10 +959,12 @@ public final class FilterUtil {
}
int keyOrdinalOfDimensionFromCurrentBlock = dimensionFromCurrentBlock.getKeyOrdinal();
for (DimColumnFilterInfo info : values) {
- if (endKey[keyOrdinalOfDimensionFromCurrentBlock] > info.getFilterList()
- .get(info.getFilterList().size() - 1)) {
- endKey[keyOrdinalOfDimensionFromCurrentBlock] =
- info.getFilterList().get(info.getFilterList().size() - 1);
+ if (keyOrdinalOfDimensionFromCurrentBlock < endKey.length) {
+ if (endKey[keyOrdinalOfDimensionFromCurrentBlock] > info.getFilterList()
+ .get(info.getFilterList().size() - 1)) {
+ endKey[keyOrdinalOfDimensionFromCurrentBlock] =
+ info.getFilterList().get(info.getFilterList().size() - 1);
+ }
}
}
long[] newEndKey = new long[endKey.length];
@@ -1040,44 +1057,43 @@ public final class FilterUtil {
*/
public static IndexKey prepareDefaultEndIndexKey(SegmentProperties segmentProperties)
throws KeyGenException {
- long[] dictionarySurrogateKey =
- new long[segmentProperties.getDimensions().size() - segmentProperties
- .getNumberOfNoDictionaryDimension()];
+ long[] dictionarySurrogateKey = new long[segmentProperties.getNumberOfDictSortColumns()];
int index = 0;
int[] dimColumnsCardinality = segmentProperties.getDimColumnsCardinality();
- for (int i = 0; i < dimColumnsCardinality.length; i++) {
+ for (int i = 0; i < dictionarySurrogateKey.length; i++) {
dictionarySurrogateKey[index++] = dimColumnsCardinality[i];
}
IndexKey endIndexKey;
byte[] dictionaryendMdkey =
- segmentProperties.getDimensionKeyGenerator().generateKey(dictionarySurrogateKey);
+ segmentProperties.getSortColumnsGenerator().generateKey(dictionarySurrogateKey);
byte[] noDictionaryEndKeyBuffer = getNoDictionaryDefaultEndKey(segmentProperties);
endIndexKey = new IndexKey(dictionaryendMdkey, noDictionaryEndKeyBuffer);
return endIndexKey;
}
public static byte[] getNoDictionaryDefaultEndKey(SegmentProperties segmentProperties) {
+
+ int numberOfNoDictionaryDimension = segmentProperties.getNumberOfNoDictSortColumns();
// in case of non filter query when no dictionary columns are present we
// need to set the default end key, as for non filter query
// we need to get the last
// block of the btree so we are setting the max byte value in the end key
ByteBuffer noDictionaryEndKeyBuffer = ByteBuffer.allocate(
- (segmentProperties.getNumberOfNoDictionaryDimension()
- * CarbonCommonConstants.SHORT_SIZE_IN_BYTE) + segmentProperties
- .getNumberOfNoDictionaryDimension());
+ (numberOfNoDictionaryDimension * CarbonCommonConstants.SHORT_SIZE_IN_BYTE)
+ + numberOfNoDictionaryDimension);
// end key structure will be
//<Offset of first No Dictionary key in 2 Bytes><Offset of second No Dictionary key in 2 Bytes>
//<Offset of n No Dictionary key in 2 Bytes><first no dictionary column value>
// <second no dictionary column value> <N no dictionary column value>
//example if we have 2 no dictionary column
//<[0,4,0,5,127,127]>
- short startPoint = (short) (segmentProperties.getNumberOfNoDictionaryDimension()
- * CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
- for (int i = 0; i < segmentProperties.getNumberOfNoDictionaryDimension(); i++) {
+ short startPoint =
+ (short) (numberOfNoDictionaryDimension * CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
+ for (int i = 0; i < numberOfNoDictionaryDimension; i++) {
noDictionaryEndKeyBuffer.putShort((startPoint));
startPoint++;
}
- for (int i = 0; i < segmentProperties.getNumberOfNoDictionaryDimension(); i++) {
+ for (int i = 0; i < numberOfNoDictionaryDimension; i++) {
noDictionaryEndKeyBuffer.put((byte) 127);
}
return noDictionaryEndKeyBuffer.array();
@@ -1094,11 +1110,9 @@ public final class FilterUtil {
public static IndexKey prepareDefaultStartIndexKey(SegmentProperties segmentProperties)
throws KeyGenException {
IndexKey startIndexKey;
- long[] dictionarySurrogateKey =
- new long[segmentProperties.getDimensions().size() - segmentProperties
- .getNumberOfNoDictionaryDimension()];
+ long[] dictionarySurrogateKey = new long[segmentProperties.getNumberOfDictSortColumns()];
byte[] dictionaryStartMdkey =
- segmentProperties.getDimensionKeyGenerator().generateKey(dictionarySurrogateKey);
+ segmentProperties.getSortColumnsGenerator().generateKey(dictionarySurrogateKey);
byte[] noDictionaryStartKeyArray = getNoDictionaryDefaultStartKey(segmentProperties);
startIndexKey = new IndexKey(dictionaryStartMdkey, noDictionaryStartKeyArray);
@@ -1106,26 +1120,27 @@ public final class FilterUtil {
}
public static byte[] getNoDictionaryDefaultStartKey(SegmentProperties segmentProperties) {
+
+ int numberOfNoDictionaryDimension = segmentProperties.getNumberOfNoDictSortColumns();
// in case of non filter query when no dictionary columns are present we
// need to set the default start key, as for non filter query we need to get the first
// block of the btree so we are setting the least byte value in the start key
ByteBuffer noDictionaryStartKeyBuffer = ByteBuffer.allocate(
- (segmentProperties.getNumberOfNoDictionaryDimension()
- * CarbonCommonConstants.SHORT_SIZE_IN_BYTE) + segmentProperties
- .getNumberOfNoDictionaryDimension());
+ (numberOfNoDictionaryDimension * CarbonCommonConstants.SHORT_SIZE_IN_BYTE)
+ + numberOfNoDictionaryDimension);
// end key structure will be
//<Offset of first No Dictionary key in 2 Bytes><Offset of second No Dictionary key in 2 Bytes>
//<Offset of n No Dictionary key in 2 Bytes><first no dictionary column value>
// <second no dictionary column value> <N no dictionary column value>
//example if we have 2 no dictionary column
//<[0,4,0,5,0,0]>
- short startPoint = (short) (segmentProperties.getNumberOfNoDictionaryDimension()
- * CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
- for (int i = 0; i < segmentProperties.getNumberOfNoDictionaryDimension(); i++) {
+ short startPoint =
+ (short) (numberOfNoDictionaryDimension * CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
+ for (int i = 0; i < numberOfNoDictionaryDimension; i++) {
noDictionaryStartKeyBuffer.putShort((startPoint));
startPoint++;
}
- for (int i = 0; i < segmentProperties.getNumberOfNoDictionaryDimension(); i++) {
+ for (int i = 0; i < numberOfNoDictionaryDimension; i++) {
noDictionaryStartKeyBuffer.put((byte) 0);
}
return noDictionaryStartKeyBuffer.array();
@@ -1181,8 +1196,8 @@ public final class FilterUtil {
FilterResolverIntf filterResolver, List<IndexKey> listOfStartEndKeys) {
IndexKey searchStartKey = null;
IndexKey searchEndKey = null;
- long[] startKey = new long[segmentProperties.getDimensionKeyGenerator().getDimCount()];
- long[] endKey = new long[segmentProperties.getDimensionKeyGenerator().getDimCount()];
+ long[] startKey = new long[segmentProperties.getNumberOfDictSortColumns()];
+ long[] endKey = new long[segmentProperties.getNumberOfDictSortColumns()];
List<byte[]> listOfStartKeyByteArray =
new ArrayList<byte[]>(segmentProperties.getNumberOfNoDictionaryDimension());
List<byte[]> listOfEndKeyByteArray =
@@ -1234,12 +1249,23 @@ public final class FilterUtil {
pruneStartAndEndKeys(setOfStartKeyByteArray, listOfStartKeyByteArray);
pruneStartAndEndKeys(setOfEndKeyByteArray, listOfEndKeyByteArray);
+ if (segmentProperties.getNumberOfNoDictSortColumns() == 0) {
+ listOfStartKeyByteArray = new ArrayList<byte[]>();
+ listOfEndKeyByteArray = new ArrayList<byte[]>();
+ } else if (segmentProperties.getNumberOfNoDictSortColumns() < listOfStartKeyByteArray
+ .size()) {
+ while (segmentProperties.getNumberOfNoDictSortColumns() < listOfStartKeyByteArray.size()) {
+ listOfStartKeyByteArray.remove(listOfStartKeyByteArray.size() - 1);
+ listOfEndKeyByteArray.remove(listOfEndKeyByteArray.size() - 1);
+ }
+ }
+
searchStartKey = FilterUtil
- .createIndexKeyFromResolvedFilterVal(startKey, segmentProperties.getDimensionKeyGenerator(),
+ .createIndexKeyFromResolvedFilterVal(startKey, segmentProperties.getSortColumnsGenerator(),
FilterUtil.getKeyWithIndexesAndValues(listOfStartKeyByteArray));
searchEndKey = FilterUtil
- .createIndexKeyFromResolvedFilterVal(endKey, segmentProperties.getDimensionKeyGenerator(),
+ .createIndexKeyFromResolvedFilterVal(endKey, segmentProperties.getSortColumnsGenerator(),
FilterUtil.getKeyWithIndexesAndValues(listOfEndKeyByteArray));
listOfStartEndKeys.add(searchStartKey);
listOfStartEndKeys.add(searchEndKey);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
index 99f5700..6341c3b 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
@@ -153,7 +153,7 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
CarbonDimension currentBlockDimension =
segmentProperties.getDimensions().get(dimensionBlocksIndex[0]);
defaultValue = FilterUtil.getMaskKey(key, currentBlockDimension,
- this.segmentProperties.getDimensionKeyGenerator());
+ this.segmentProperties.getSortColumnsGenerator());
}
if (dimensionColumnDataChunk.isExplicitSorted()) {
return setFilterdIndexToBitSetWithColumnIndex(dimensionColumnDataChunk, numerOfRows,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
index 1883607..bfe4ed1 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
@@ -153,7 +153,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
CarbonDimension currentBlockDimension =
segmentProperties.getDimensions().get(dimensionBlocksIndex[0]);
defaultValue = FilterUtil.getMaskKey(key, currentBlockDimension,
- this.segmentProperties.getDimensionKeyGenerator());
+ this.segmentProperties.getSortColumnsGenerator());
}
if (dimensionColumnDataChunk.isExplicitSorted()) {
return setFilterdIndexToBitSetWithColumnIndex(dimensionColumnDataChunk, numerOfRows,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java
index 351d2c0..efedd93 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java
@@ -67,7 +67,8 @@ public class NoDictionaryTypeVisitor implements ResolvedFilterInfoVisitorIntf {
throw new FilterUnsupportedException(e);
}
resolvedFilterObject = FilterUtil
- .getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, metadata.isIncludeFilter());
+ .getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, metadata.isIncludeFilter(),
+ metadata.getColumnExpression().getDataType());
visitableObj.setFilterValues(resolvedFilterObject);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeNoDictionaryTypeVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeNoDictionaryTypeVisitor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeNoDictionaryTypeVisitor.java
index 8d0a8b4..d703ed0 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeNoDictionaryTypeVisitor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeNoDictionaryTypeVisitor.java
@@ -72,7 +72,8 @@ public class RangeNoDictionaryTypeVisitor extends NoDictionaryTypeVisitor
throw new FilterUnsupportedException(e);
}
resolvedFilterObject = FilterUtil
- .getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, metadata.isIncludeFilter());
+ .getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, metadata.isIncludeFilter(),
+ metadata.getColumnExpression().getDataType());
visitableObj.setFilterValues(resolvedFilterObject);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
index 4e5681c..cdab4e4 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
@@ -104,7 +104,9 @@ public abstract class AbstractDetailQueryResultIterator<E> extends CarbonIterato
private void intialiseInfos() {
for (BlockExecutionInfo blockInfo : blockExecutionInfos) {
- DataRefNodeFinder finder = new BTreeDataRefNodeFinder(blockInfo.getEachColumnValueSize());
+ DataRefNodeFinder finder = new BTreeDataRefNodeFinder(blockInfo.getEachColumnValueSize(),
+ blockInfo.getDataBlock().getSegmentProperties().getNumberOfSortColumns(),
+ blockInfo.getDataBlock().getSegmentProperties().getNumberOfNoDictSortColumns());
DataRefNode startDataBlock = finder
.findFirstDataBlock(blockInfo.getDataBlock().getDataRefNode(), blockInfo.getStartKey());
while (startDataBlock.nodeNumber() < blockInfo.getStartBlockletIndex()) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnVector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnVector.java b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnVector.java
index 4952e07..82a0b45 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnVector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnVector.java
@@ -17,10 +17,15 @@
package org.apache.carbondata.core.scan.result.vector;
+import org.apache.spark.sql.types.DataType;
import org.apache.spark.sql.types.Decimal;
public interface CarbonColumnVector {
+ void putBoolean(int rowId, boolean value);
+
+ void putFloat(int rowId, float value);
+
void putShort(int rowId, short value);
void putShorts(int rowId, int count, short value);
@@ -59,4 +64,6 @@ public interface CarbonColumnVector {
void reset();
+ DataType getType();
+
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
index 48a081d..52f2f31 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
@@ -21,6 +21,7 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.BitSet;
import java.util.List;
+import java.util.Map;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.BlockInfo;
@@ -195,6 +196,12 @@ public abstract class AbstractDataFileFooterConverter {
wrapperColumnSchema.setColumnGroup(externalColumnSchema.getColumn_group_id());
wrapperColumnSchema.setScale(externalColumnSchema.getScale());
wrapperColumnSchema.setDefaultValue(externalColumnSchema.getDefault_value());
+ Map<String, String> properties = externalColumnSchema.getColumnProperties();
+ if (properties != null) {
+ if (properties.get(CarbonCommonConstants.SORT_COLUMNS) != null) {
+ wrapperColumnSchema.setSortColumn(true);
+ }
+ }
return wrapperColumnSchema;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
index 16c6760..f3f0fcd 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
@@ -17,7 +17,11 @@
package org.apache.carbondata.core.util;
+import java.io.UnsupportedEncodingException;
+import java.math.BigDecimal;
+import java.math.BigInteger;
import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.memory.CarbonUnsafe;
@@ -27,7 +31,13 @@ import org.apache.carbondata.core.memory.CarbonUnsafe;
*/
public final class ByteUtil {
- private static final int SIZEOF_LONG = 8;
+ public static final int SIZEOF_LONG = 8;
+
+ public static final int SIZEOF_INT = 4;
+
+ public static final int SIZEOF_SHORT = 2;
+
+ public static final String UTF8_CSN = StandardCharsets.UTF_8.name();
private ByteUtil() {
@@ -326,4 +336,328 @@ public final class ByteUtil {
}
+ /**
+ * Stirng => byte[]
+ *
+ * @param s
+ * @return
+ */
+ public static byte[] toBytes(String s) {
+ try {
+ return s.getBytes(UTF8_CSN);
+ } catch (UnsupportedEncodingException e) {
+ // should never happen!
+ throw new IllegalArgumentException("UTF8 decoding is not supported", e);
+ }
+ }
+
+ /**
+ * byte[] => String
+ *
+ * @param b
+ * @param off
+ * @param len
+ * @return
+ */
+ public static String toString(final byte[] b, int off, int len) {
+ if (b == null) {
+ return null;
+ }
+ if (len == 0) {
+ return "";
+ }
+ try {
+ return new String(b, off, len, UTF8_CSN);
+ } catch (UnsupportedEncodingException e) {
+ // should never happen!
+ throw new IllegalArgumentException("UTF8 encoding is not supported", e);
+ }
+ }
+
+ /**
+ * boolean => byte[]
+ *
+ * @param b
+ * @return
+ */
+ public static byte[] toBytes(final boolean b) {
+ return new byte[] { b ? (byte) -1 : (byte) 0 };
+ }
+
+ /**
+ * byte[] => boolean
+ *
+ * @param b
+ * @return
+ */
+ public static boolean toBoolean(final byte[] b) {
+ if (b.length != 1) {
+ throw new IllegalArgumentException("Array has wrong size: " + b.length);
+ }
+ return b[0] != (byte) 0;
+ }
+
+ /**
+ * short => byte[]
+ *
+ * @param val
+ * @return
+ */
+ public static byte[] toBytes(short val) {
+ byte[] b = new byte[SIZEOF_SHORT];
+ b[1] = (byte) val;
+ val >>= 8;
+ b[0] = (byte) val;
+ return b;
+ }
+
+ /**
+ * byte[] => short
+ *
+ * @param bytes
+ * @param offset
+ * @param length
+ * @return
+ */
+ public static short toShort(byte[] bytes, int offset, final int length) {
+ if (length != SIZEOF_SHORT || offset + length > bytes.length) {
+ throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_SHORT);
+ }
+ if (CarbonUnsafe.unsafe != null) {
+ if (CarbonUnsafe.ISLITTLEENDIAN) {
+ return Short.reverseBytes(
+ CarbonUnsafe.unsafe.getShort(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET));
+ } else {
+ return CarbonUnsafe.unsafe.getShort(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
+ }
+ } else {
+ short n = 0;
+ n ^= bytes[offset] & 0xFF;
+ n <<= 8;
+ n ^= bytes[offset + 1] & 0xFF;
+ return n;
+ }
+ }
+
+ /**
+ * int => byte[]
+ *
+ * @param val
+ * @return
+ */
+ public static byte[] toBytes(int val) {
+ byte[] b = new byte[4];
+ for (int i = 3; i > 0; i--) {
+ b[i] = (byte) val;
+ val >>>= 8;
+ }
+ b[0] = (byte) val;
+ return b;
+ }
+
+ /**
+ * byte[] => int
+ *
+ * @param bytes
+ * @param offset
+ * @param length
+ * @return
+ */
+ public static int toInt(byte[] bytes, int offset, final int length) {
+ if (length != SIZEOF_INT || offset + length > bytes.length) {
+ throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_INT);
+ }
+ if (CarbonUnsafe.unsafe != null) {
+ if (CarbonUnsafe.ISLITTLEENDIAN) {
+ return Integer.reverseBytes(
+ CarbonUnsafe.unsafe.getInt(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET));
+ } else {
+ return CarbonUnsafe.unsafe.getInt(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
+ }
+ } else {
+ int n = 0;
+ for (int i = offset; i < (offset + length); i++) {
+ n <<= 8;
+ n ^= bytes[i] & 0xFF;
+ }
+ return n;
+ }
+ }
+
+ /**
+ * float => byte[]
+ *
+ * @param f
+ * @return
+ */
+ public static byte[] toBytes(final float f) {
+ // Encode it as int
+ return toBytes(Float.floatToRawIntBits(f));
+ }
+
+ /**
+ * byte[] => float
+ *
+ * @param bytes
+ * @param offset
+ * @return
+ */
+ public static float toFloat(byte[] bytes, int offset) {
+ return Float.intBitsToFloat(toInt(bytes, offset, SIZEOF_INT));
+ }
+
+ /**
+ * long => byte[]
+ *
+ * @param val
+ * @return
+ */
+ public static byte[] toBytes(long val) {
+ byte[] b = new byte[8];
+ for (int i = 7; i > 0; i--) {
+ b[i] = (byte) val;
+ val >>>= 8;
+ }
+ b[0] = (byte) val;
+ return b;
+ }
+
+ /**
+ * byte[] => long
+ */
+ public static long toLong(byte[] bytes, int offset, final int length) {
+ if (length != SIZEOF_LONG || offset + length > bytes.length) {
+ throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_LONG);
+ }
+ if (CarbonUnsafe.unsafe != null) {
+ if (CarbonUnsafe.ISLITTLEENDIAN) {
+ return Long.reverseBytes(
+ CarbonUnsafe.unsafe.getLong(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET));
+ } else {
+ return CarbonUnsafe.unsafe.getLong(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
+ }
+ } else {
+ long l = 0;
+ for (int i = offset; i < offset + length; i++) {
+ l <<= 8;
+ l ^= bytes[i] & 0xFF;
+ }
+ return l;
+ }
+ }
+
+ /**
+ * doube => byte[]
+ *
+ * @param d
+ * @return
+ */
+ public static byte[] toBytes(final double d) {
+ // Encode it as a long
+ return toBytes(Double.doubleToRawLongBits(d));
+ }
+
+ /**
+ * byte[] => double
+ *
+ * @param bytes
+ * @param offset
+ * @return
+ */
+ public static double toDouble(final byte[] bytes, final int offset) {
+ return Double.longBitsToDouble(toLong(bytes, offset, SIZEOF_LONG));
+ }
+
+ /**
+ * BigDecimal => byte[]
+ *
+ * @param val
+ * @return
+ */
+ public static byte[] toBytes(BigDecimal val) {
+ byte[] valueBytes = val.unscaledValue().toByteArray();
+ byte[] result = new byte[valueBytes.length + SIZEOF_INT];
+ int offset = putInt(result, 0, val.scale());
+ putBytes(result, offset, valueBytes, 0, valueBytes.length);
+ return result;
+ }
+
+ /**
+ * byte[] => BigDecimal
+ *
+ * @param bytes
+ * @param offset
+ * @param length
+ * @return
+ */
+ public static BigDecimal toBigDecimal(byte[] bytes, int offset, final int length) {
+ if (bytes == null || length < SIZEOF_INT + 1 || (offset + length > bytes.length)) {
+ return null;
+ }
+
+ int scale = toInt(bytes, offset, bytes.length);
+ byte[] tcBytes = new byte[length - SIZEOF_INT];
+ System.arraycopy(bytes, offset + SIZEOF_INT, tcBytes, 0, length - SIZEOF_INT);
+ return new BigDecimal(new BigInteger(tcBytes), scale);
+ }
+
+ private static IllegalArgumentException explainWrongLengthOrOffset(final byte[] bytes,
+ final int offset, final int length, final int expectedLength) {
+ String reason;
+ if (length != expectedLength) {
+ reason = "Wrong length: " + length + ", expected " + expectedLength;
+ } else {
+ reason = "offset (" + offset + ") + length (" + length + ") exceed the"
+ + " capacity of the array: " + bytes.length;
+ }
+ return new IllegalArgumentException(reason);
+ }
+
+ /**
+ * Put an int value out to the specified byte array position.
+ *
+ * @param bytes the byte array
+ * @param offset position in the array
+ * @param val int to write out
+ * @return incremented offset
+ * @throws IllegalArgumentException if the byte array given doesn't have
+ * enough room at the offset specified.
+ */
+ public static int putInt(byte[] bytes, int offset, int val) {
+ if (bytes.length - offset < SIZEOF_INT) {
+ throw new IllegalArgumentException(
+ "Not enough room to put an int at" + " offset " + offset + " in a " + bytes.length
+ + " byte array");
+ }
+ if (CarbonUnsafe.unsafe != null) {
+ if (CarbonUnsafe.ISLITTLEENDIAN) {
+ val = Integer.reverseBytes(val);
+ }
+ CarbonUnsafe.unsafe.putInt(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET, val);
+ return offset + ByteUtil.SIZEOF_INT;
+ } else {
+ for (int i = offset + 3; i > offset; i--) {
+ bytes[i] = (byte) val;
+ val >>>= 8;
+ }
+ bytes[offset] = (byte) val;
+ return offset + SIZEOF_INT;
+ }
+ }
+
+ /**
+ * Put bytes at the specified byte array position.
+ *
+ * @param tgtBytes the byte array
+ * @param tgtOffset position in the array
+ * @param srcBytes array to write out
+ * @param srcOffset source offset
+ * @param srcLength source length
+ * @return incremented offset
+ */
+ public static int putBytes(byte[] tgtBytes, int tgtOffset, byte[] srcBytes, int srcOffset,
+ int srcLength) {
+ System.arraycopy(srcBytes, srcOffset, tgtBytes, tgtOffset, srcLength);
+ return tgtOffset + srcLength;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 8956549..95ee94b 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -641,6 +641,24 @@ public final class CarbonUtil {
}
/**
+ * remove the quote char for a string, e.g. "abc" => abc, 'abc' => abc
+ * @param parseStr
+ * @return
+ */
+ public static String unquoteChar(String parseStr) {
+ if (parseStr == null) {
+ return null;
+ }
+ if (parseStr.startsWith("'") && parseStr.endsWith("'")) {
+ return parseStr.substring(1, parseStr.length() - 1);
+ } else if (parseStr.startsWith("\"") && parseStr.endsWith("\"")) {
+ return parseStr.substring(1, parseStr.length() - 1);
+ } else {
+ return parseStr;
+ }
+ }
+
+ /**
* special char delimiter Converter
*
* @param delimiter
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
index d8e0c82..2e16b0f 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -323,6 +323,76 @@ public final class DataTypeUtil {
}
+ public static byte[] getBytesBasedOnDataTypeForNoDictionaryColumn(String dimensionValue,
+ DataType actualDataType) throws Throwable {
+ switch (actualDataType) {
+ case STRING:
+ return ByteUtil.toBytes(dimensionValue);
+ case BOOLEAN:
+ return ByteUtil.toBytes(Boolean.parseBoolean(dimensionValue));
+ case SHORT:
+ return ByteUtil.toBytes(Short.parseShort(dimensionValue));
+ case INT:
+ return ByteUtil.toBytes(Integer.parseInt(dimensionValue));
+ case FLOAT:
+ return ByteUtil.toBytes(Float.parseFloat(dimensionValue));
+ case LONG:
+ return ByteUtil.toBytes(Long.parseLong(dimensionValue));
+ case DOUBLE:
+ return ByteUtil.toBytes(Double.parseDouble(dimensionValue));
+ case DECIMAL:
+ return ByteUtil.toBytes(new BigDecimal(dimensionValue));
+ default:
+ return ByteUtil.toBytes(dimensionValue);
+ }
+ }
+
+
+ /**
+ * Below method will be used to convert the data passed to its actual data
+ * type
+ *
+ * @param dataInBytes data
+ * @param actualDataType actual data type
+ * @return actual data after conversion
+ */
+ public static Object getDataBasedOnDataTypeForNoDictionaryColumn(byte[] dataInBytes,
+ DataType actualDataType) {
+ if (null == dataInBytes || Arrays
+ .equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY, dataInBytes)) {
+ return null;
+ }
+ try {
+ switch (actualDataType) {
+ case STRING:
+ return UTF8String.fromBytes(dataInBytes);
+ case BOOLEAN:
+ return ByteUtil.toBoolean(dataInBytes);
+ case SHORT:
+ return ByteUtil.toShort(dataInBytes, 0, dataInBytes.length);
+ case INT:
+ return ByteUtil.toInt(dataInBytes, 0, dataInBytes.length);
+ case FLOAT:
+ return ByteUtil.toFloat(dataInBytes, 0);
+ case LONG:
+ return ByteUtil.toLong(dataInBytes, 0, dataInBytes.length);
+ case DOUBLE:
+ return ByteUtil.toDouble(dataInBytes, 0);
+ case DECIMAL:
+ return ByteUtil.toBigDecimal(dataInBytes, 0, dataInBytes.length);
+ default:
+ return ByteUtil.toString(dataInBytes, 0, dataInBytes.length);
+ }
+ } catch (Throwable ex) {
+ String data = new String(dataInBytes, CarbonCommonConstants.DEFAULT_CHARSET_CLASS);
+ LOGGER.error("Cannot convert" + data + " to " + actualDataType.getName() + " type value" + ex
+ .getMessage());
+ LOGGER.error("Problem while converting data type" + data);
+ return null;
+ }
+ }
+
+
/**
* Below method will be used to convert the data passed to its actual data
* type
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java
index 54bbe06..d6081a3 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java
@@ -68,7 +68,7 @@ public class BTreeBlockFinderTest extends TestCase {
builder.build(infos);
DataRefNode dataBlock = builder.get();
assertTrue(dataBlock != null);
- DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { -1 });
+ DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { -1 }, 1, 1);
ByteBuffer buffer = ByteBuffer.allocate(4 + 2);
buffer.rewind();
buffer.putShort((short) 1);
@@ -88,7 +88,7 @@ public class BTreeBlockFinderTest extends TestCase {
builder.build(infos);
DataRefNode dataBlock = builder.get();
assertTrue(dataBlock != null);
- DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { -1 });
+ DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { -1 }, 1, 1);
ByteBuffer buffer = ByteBuffer.allocate(4 + 1);
buffer.rewind();
buffer.put((byte) 1);
@@ -109,7 +109,7 @@ public class BTreeBlockFinderTest extends TestCase {
builder.build(infos);
DataRefNode dataBlock = builder.get();
assertTrue(dataBlock != null);
- DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 });
+ DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 }, 2, 0);
int[] dimensionBitLength =
CarbonUtil.getDimensionBitLength(new int[] { 10000, 10000 }, new int[] { 1, 1 });
KeyGenerator multiDimKeyVarLengthGenerator =
@@ -131,7 +131,7 @@ public class BTreeBlockFinderTest extends TestCase {
builder.build(infos);
DataRefNode dataBlock = builder.get();
assertTrue(dataBlock != null);
- DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 });
+ DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 }, 2, 0);
int[] dimensionBitLength =
CarbonUtil.getDimensionBitLength(new int[] { 10000, 10000 }, new int[] { 1, 1 });
KeyGenerator multiDimKeyVarLengthGenerator =
@@ -159,7 +159,7 @@ public class BTreeBlockFinderTest extends TestCase {
builder.build(infos);
DataRefNode dataBlock = builder.get();
assertTrue(dataBlock != null);
- DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 });
+ DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 }, 2, 0);
int[] dimensionBitLength =
CarbonUtil.getDimensionBitLength(new int[] { 10000, 10000 }, new int[] { 1, 1 });
KeyGenerator multiDimKeyVarLengthGenerator =
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java b/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java
index fee3d3d..078479f 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java
@@ -37,6 +37,7 @@ import org.apache.carbondata.core.scan.expression.ColumnExpression;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.expression.LiteralExpression;
import org.apache.carbondata.core.scan.expression.conditional.ListExpression;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.intf.RowImpl;
import org.apache.carbondata.core.util.BitSetGroup;
@@ -363,7 +364,7 @@ public class FilterUtilTest extends AbstractDictionaryCacheTest {
assertFalse(result);
}
- @Test public void testGetNoDictionaryValKeyMemberForFilter() {
+ @Test public void testGetNoDictionaryValKeyMemberForFilter() throws FilterUnsupportedException {
boolean isIncludeFilter = true;
AbsoluteTableIdentifier absoluteTableIdentifier =
new AbsoluteTableIdentifier(this.carbonStorePath, carbonTableIdentifier);
@@ -371,7 +372,9 @@ public class FilterUtilTest extends AbstractDictionaryCacheTest {
List<String> evaluateResultListFinal = new ArrayList<>();
evaluateResultListFinal.add("test1");
evaluateResultListFinal.add("test2");
- assertTrue(FilterUtil.getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, isIncludeFilter) instanceof DimColumnFilterInfo);
+ assertTrue(FilterUtil
+ .getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, isIncludeFilter,
+ DataType.STRING) instanceof DimColumnFilterInfo);
}
@Test public void testPrepareDefaultStartIndexKey() throws KeyGenException {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
index cda34e4..05f2b67 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
@@ -598,7 +598,9 @@ public class CarbonInputFormat<T> extends FileInputFormat<Void, T> {
// Add all blocks of btree into result
DataRefNodeFinder blockFinder =
- new BTreeDataRefNodeFinder(segmentProperties.getEachDimColumnValueSize());
+ new BTreeDataRefNodeFinder(segmentProperties.getEachDimColumnValueSize(),
+ segmentProperties.getNumberOfSortColumns(),
+ segmentProperties.getNumberOfNoDictSortColumns());
DataRefNode startBlock =
blockFinder.findFirstDataBlock(abstractIndex.getDataRefNode(), startIndexKey);
DataRefNode endBlock =
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
index 7ba6133..d476f74 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
@@ -199,7 +199,9 @@ class InMemoryBTreeIndex implements Index {
// Add all blocks of btree into result
DataRefNodeFinder blockFinder =
- new BTreeDataRefNodeFinder(segmentProperties.getEachDimColumnValueSize());
+ new BTreeDataRefNodeFinder(segmentProperties.getEachDimColumnValueSize(),
+ segmentProperties.getNumberOfSortColumns(),
+ segmentProperties.getNumberOfNoDictSortColumns());
DataRefNode startBlock =
blockFinder.findFirstDataBlock(abstractIndex.getDataRefNode(), startIndexKey);
DataRefNode endBlock =
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/DictionaryDecodeReadSupport.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/DictionaryDecodeReadSupport.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/DictionaryDecodeReadSupport.java
index 79a1559..0239bce 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/DictionaryDecodeReadSupport.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/DictionaryDecodeReadSupport.java
@@ -87,6 +87,9 @@ public class DictionaryDecodeReadSupport<T> implements CarbonReadSupport<T> {
* threshold is reached
*/
@Override public void close() {
+ if (dictionaries == null) {
+ return;
+ }
for (int i = 0; i < dictionaries.length; i++) {
CarbonUtil.clearDictionaryCache(dictionaries[i]);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java
index e39d214..b814468 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java
@@ -53,7 +53,9 @@ public class BlockLevelTraverser {
FilterUtil.prepareDefaultStartIndexKey(abstractIndex.getSegmentProperties());
DataRefNodeFinder blockFinder = new BTreeDataRefNodeFinder(
- abstractIndex.getSegmentProperties().getEachDimColumnValueSize());
+ abstractIndex.getSegmentProperties().getEachDimColumnValueSize(),
+ abstractIndex.getSegmentProperties().getNumberOfSortColumns(),
+ abstractIndex.getSegmentProperties().getNumberOfNoDictSortColumns());
DataRefNode currentBlock =
blockFinder.findFirstDataBlock(abstractIndex.getDataRefNode(), searchStartKey);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
----------------------------------------------------------------------
diff --git a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
index c90b34c..1448957 100644
--- a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
+++ b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
@@ -203,6 +203,7 @@ public class StoreCreator {
date.setColumnUniqueId(UUID.randomUUID().toString());
date.setDimensionColumn(true);
date.setColumnGroup(2);
+ date.setSortColumn(true);
columnSchemas.add(date);
ColumnSchema country = new ColumnSchema();
@@ -213,6 +214,7 @@ public class StoreCreator {
country.setColumnUniqueId(UUID.randomUUID().toString());
country.setDimensionColumn(true);
country.setColumnGroup(3);
+ country.setSortColumn(true);
columnSchemas.add(country);
ColumnSchema name = new ColumnSchema();
@@ -223,6 +225,7 @@ public class StoreCreator {
name.setColumnUniqueId(UUID.randomUUID().toString());
name.setDimensionColumn(true);
name.setColumnGroup(4);
+ name.setSortColumn(true);
columnSchemas.add(name);
ColumnSchema phonetype = new ColumnSchema();
@@ -233,6 +236,7 @@ public class StoreCreator {
phonetype.setColumnUniqueId(UUID.randomUUID().toString());
phonetype.setDimensionColumn(true);
phonetype.setColumnGroup(5);
+ phonetype.setSortColumn(true);
columnSchemas.add(phonetype);
ColumnSchema serialname = new ColumnSchema();
@@ -243,6 +247,7 @@ public class StoreCreator {
serialname.setColumnUniqueId(UUID.randomUUID().toString());
serialname.setDimensionColumn(true);
serialname.setColumnGroup(6);
+ serialname.setSortColumn(true);
columnSchemas.add(serialname);
ColumnSchema salary = new ColumnSchema();
[08/19] carbondata git commit: [CARBONDATA-888] Added options to
include and exclude dictionary columns in dataframe
Posted by ra...@apache.org.
[CARBONDATA-888] Added options to include and exclude dictionary columns in dataframe
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/f2a51767
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/f2a51767
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/f2a51767
Branch: refs/heads/master
Commit: f2a517679e753b63325f8d8fcc0f33f3d76d0e79
Parents: f5c6f28
Author: Sanoj MG <sa...@gmail.com>
Authored: Tue Apr 18 17:22:14 2017 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Sun May 14 20:40:12 2017 +0800
----------------------------------------------------------------------
.../testsuite/dataload/TestLoadDataFrame.scala | 47 +++++++++++++++++++-
.../spark/CarbonDataFrameWriter.scala | 5 +++
.../spark/sql/CarbonDataFrameWriter.scala | 29 +++---------
3 files changed, 56 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f2a51767/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala
index 2d86497..f50620f 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala
@@ -27,6 +27,7 @@ import org.scalatest.BeforeAndAfterAll
class TestLoadDataFrame extends QueryTest with BeforeAndAfterAll {
var df: DataFrame = _
var dataFrame: DataFrame = _
+ var df2: DataFrame = _
def buildTestData() = {
@@ -45,6 +46,9 @@ class TestLoadDataFrame extends QueryTest with BeforeAndAfterAll {
StructField("string", StringType, nullable = false) :: Nil)
dataFrame = sqlContext.createDataFrame(rdd, schema)
+ df2 = sqlContext.sparkContext.parallelize(1 to 1000)
+ .map(x => ("key_" + x, "str_" + x, x, x * 2, x * 3))
+ .toDF("c1", "c2", "c3", "c4", "c5")
}
def dropTable() = {
@@ -52,7 +56,9 @@ class TestLoadDataFrame extends QueryTest with BeforeAndAfterAll {
sql("DROP TABLE IF EXISTS carbon2")
sql("DROP TABLE IF EXISTS carbon3")
sql("DROP TABLE IF EXISTS carbon4")
-
+ sql("DROP TABLE IF EXISTS carbon5")
+ sql("DROP TABLE IF EXISTS carbon6")
+ sql("DROP TABLE IF EXISTS carbon7")
}
@@ -121,6 +127,45 @@ class TestLoadDataFrame extends QueryTest with BeforeAndAfterAll {
sql("SELECT count(*) FROM carbon2"),Seq(Row(32000)))
}
+ test("test load dataframe with integer columns included in the dictionary"){
+ df2.write
+ .format("carbondata")
+ .option("tableName", "carbon5")
+ .option("compress", "true")
+ .option("dictionary_include","c3,c4")
+ .mode(SaveMode.Overwrite)
+ .save()
+ checkAnswer(
+ sql("select count(*) from carbon5 where c3 > 300"), Row(700)
+ )
+ }
+
+ test("test load dataframe with string column excluded from the dictionary"){
+ df2.write
+ .format("carbondata")
+ .option("tableName", "carbon6")
+ .option("compress", "true")
+ .option("dictionary_exclude","c2")
+ .mode(SaveMode.Overwrite)
+ .save()
+ checkAnswer(
+ sql("select count(*) from carbon6 where c3 > 300"), Row(700)
+ )
+ }
+
+ test("test load dataframe with both dictionary include and exclude specified"){
+ df2.write
+ .format("carbondata")
+ .option("tableName", "carbon7")
+ .option("compress", "true")
+ .option("dictionary_include","c3,c4")
+ .option("dictionary_exclude","c2")
+ .mode(SaveMode.Overwrite)
+ .save()
+ checkAnswer(
+ sql("select count(*) from carbon7 where c3 > 300"), Row(700)
+ )
+ }
override def afterAll {
dropTable
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f2a51767/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala
index 917e2ed..36e2440 100644
--- a/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala
+++ b/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala
@@ -174,6 +174,10 @@ class CarbonDataFrameWriter(val dataFrame: DataFrame) {
}
private def makeCreateTableString(schema: StructType, options: CarbonOption): String = {
+ val properties = Map(
+ "DICTIONARY_INCLUDE" -> options.dictionaryInclude,
+ "DICTIONARY_EXCLUDE" -> options.dictionaryExclude
+ ).filter(_._2.isDefined).map(p => s"'${p._1}' = '${p._2.get}'").mkString(",")
val carbonSchema = schema.map { field =>
s"${ field.name } ${ convertToCarbonType(field.dataType) }"
}
@@ -181,6 +185,7 @@ class CarbonDataFrameWriter(val dataFrame: DataFrame) {
CREATE TABLE IF NOT EXISTS ${options.dbName}.${options.tableName}
(${ carbonSchema.mkString(", ") })
STORED BY '${ CarbonContext.datasourceName }'
+ ${ if (properties.nonEmpty) " TBLPROPERTIES (" + properties + ")" else ""}
"""
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f2a51767/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
index 4de644a..1054c62 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
@@ -161,30 +161,11 @@ class CarbonDataFrameWriter(sqlContext: SQLContext, val dataFrame: DataFrame) {
val carbonSchema = schema.map { field =>
s"${ field.name } ${ convertToCarbonType(field.dataType) }"
}
- var property = new StringBuilder
- property.append(
- if (options.dictionaryInclude.isDefined) {
- s"'DICTIONARY_INCLUDE' = '${options.dictionaryInclude.get}' ,"
- } else {
- ""
- }
- ).append(
- if (options.dictionaryExclude.isDefined) {
- s"'DICTIONARY_EXCLUDE' = '${options.dictionaryExclude.get}' ,"
- } else {
- ""
- }
- ).append(
- if (options.tableBlockSize.isDefined) {
- s"'table_blocksize' = '${options.tableBlockSize.get}'"
- } else {
- ""
- }
- )
- if (property.nonEmpty && property.charAt(property.length-1) == ',') {
- property = property.replace(property.length - 1, property.length, "")
- }
-
+ val property = Map(
+ "DICTIONARY_INCLUDE" -> options.dictionaryInclude,
+ "DICTIONARY_EXCLUDE" -> options.dictionaryExclude,
+ "TABLE_BLOCKSIZE" -> options.tableBlockSize
+ ).filter(_._2.isDefined).map(p => s"'${p._1}' = '${p._2.get}'").mkString(",")
s"""
| CREATE TABLE IF NOT EXISTS ${options.dbName}.${options.tableName}
| (${ carbonSchema.mkString(", ") })
[02/19] carbondata git commit: [CARBONDATA-782] add SORT_COLUMNS
options
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
new file mode 100644
index 0000000..88b0421
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
@@ -0,0 +1,267 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.spark.testsuite.sortcolumns
+
+import org.apache.spark.sql.common.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+
+class TestSortColumns extends QueryTest with BeforeAndAfterAll {
+
+ override def beforeAll {
+ dropTable
+
+ sql("CREATE TABLE origintable1 (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format'")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE origintable1 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ }
+
+ test("create table with no dictionary sort_columns") {
+ sql("CREATE TABLE sorttable1 (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='empno')")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE sorttable1 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ checkAnswer(sql("select empno from sorttable1"), sql("select empno from sorttable1 order by empno"))
+ }
+
+ test("create table with dictionary sort_columns") {
+ sql("CREATE TABLE sorttable2 (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='empname')")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE sorttable2 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ checkAnswer(sql("select empname from sorttable2"),sql("select empname from origintable1"))
+ }
+
+ test("create table with direct-dictioanry sort_columns") {
+ sql("CREATE TABLE sorttable3 (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='doj')")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE sorttable3 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ checkAnswer(sql("select doj from sorttable3"), sql("select doj from sorttable3 order by doj"))
+ }
+
+ test("create table with multi-sort_columns and data loading with offheap safe") {
+ try {
+ setLoadingProperties("true", "false", "false")
+ sql("CREATE TABLE sorttable4_offheap_safe (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='workgroupcategory, empname')")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE sorttable4_offheap_safe OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ checkAnswer(sql("select workgroupcategory, empname from sorttable4_offheap_safe"), sql("select workgroupcategory, empname from origintable1 order by workgroupcategory"))
+ } finally {
+ defaultLoadingProperties
+ }
+ }
+
+ test("create table with multi-sort_columns and data loading with offheap and unsafe sort") {
+ try {
+ setLoadingProperties("true", "true", "false")
+ sql(
+ "CREATE TABLE sorttable4_offheap_unsafe (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='workgroupcategory, empname')")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE sorttable4_offheap_unsafe OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ checkAnswer(sql("select workgroupcategory, empname from sorttable4_offheap_unsafe"),
+ sql("select workgroupcategory, empname from origintable1 order by workgroupcategory"))
+ } finally {
+ defaultLoadingProperties
+ }
+ }
+
+ test("create table with multi-sort_columns and data loading with offheap and inmemory sort") {
+ try {
+ setLoadingProperties("true", "false", "true")
+ sql(
+ "CREATE TABLE sorttable4_offheap_inmemory (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='workgroupcategory, empname')")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE sorttable4_offheap_inmemory OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ checkAnswer(sql("select workgroupcategory, empname from sorttable4_offheap_inmemory"),
+ sql("select workgroupcategory, empname from origintable1 order by workgroupcategory"))
+ } finally {
+ defaultLoadingProperties
+ }
+ }
+
+ test("create table with multi-sort_columns and data loading with heap") {
+ try {
+ setLoadingProperties("false", "false", "false")
+ sql(
+ "CREATE TABLE sorttable4_heap_safe (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='workgroupcategory, empname')")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE sorttable4_heap_safe OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ checkAnswer(sql("select workgroupcategory, empname from sorttable4_heap_safe"),
+ sql("select workgroupcategory, empname from origintable1 order by workgroupcategory"))
+ } finally {
+ defaultLoadingProperties
+ }
+ }
+
+ test("create table with multi-sort_columns and data loading with heap and unsafe sort") {
+ try {
+ setLoadingProperties("false", "true", "false")
+ sql(
+ "CREATE TABLE sorttable4_heap_unsafe (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='workgroupcategory, empname')")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE sorttable4_heap_unsafe OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ checkAnswer(sql("select workgroupcategory, empname from sorttable4_heap_unsafe"),
+ sql("select workgroupcategory, empname from origintable1 order by workgroupcategory"))
+ } finally {
+ defaultLoadingProperties
+ }
+ }
+
+ test("create table with multi-sort_columns and data loading with heap and inmemory sort") {
+ try {
+ setLoadingProperties("false", "false", "true")
+ sql(
+ "CREATE TABLE sorttable4_heap_inmemory (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='workgroupcategory, empname')")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE sorttable4_heap_inmemory OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ checkAnswer(sql("select workgroupcategory, empname from sorttable4_heap_inmemory"),
+ sql("select workgroupcategory, empname from origintable1 order by workgroupcategory"))
+ } finally {
+ defaultLoadingProperties
+ }
+ }
+
+ test("compaction on sort_columns table") {
+ sql("CREATE TABLE origintable2 (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format'")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE origintable2 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE origintable2 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE origintable2 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE origintable2 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ sql("alter table origintable2 compact 'minor'")
+
+ sql("CREATE TABLE sorttable5 (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='empno')")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE sorttable5 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE sorttable5 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE sorttable5 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE sorttable5 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ sql("alter table sorttable5 compact 'minor'")
+
+ checkAnswer(sql("select empno from sorttable5"), sql("select empno from origintable2 order by empno"))
+ }
+
+ test("filter on sort_columns include no-dictionary, direct-dictionary and dictioanry") {
+ sql("CREATE TABLE sorttable6 (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='workgroupcategory, doj, empname')")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE sorttable6 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ // no dictionary
+ checkAnswer(sql("select * from sorttable6 where workgroupcategory = 1"), sql("select * from origintable1 where workgroupcategory = 1 order by doj"))
+ // direct dictionary
+ checkAnswer(sql("select * from sorttable6 where doj = '2007-01-17 00:00:00'"), sql("select * from origintable1 where doj = '2007-01-17 00:00:00'"))
+ // dictionary
+ checkAnswer(sql("select * from sorttable6 where empname = 'madhan'"), sql("select * from origintable1 where empname = 'madhan'"))
+ }
+
+ test("unsorted table creation, query data loading with heap and safe sort config") {
+ try {
+ setLoadingProperties("false", "false", "false")
+ sql("CREATE TABLE unsortedtable_heap_safe (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_heap_safe OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ checkAnswer(sql("select * from unsortedtable_heap_safe where empno = 11"), sql("select * from origintable1 where empno = 11"))
+ checkAnswer(sql("select * from unsortedtable_heap_safe order by empno"), sql("select * from origintable1 order by empno"))
+ } finally {
+ defaultLoadingProperties
+ }
+ }
+
+ test("unsorted table creation, query and data loading with heap and unsafe sort config") {
+ try {
+ setLoadingProperties("false", "true", "false")
+ sql("CREATE TABLE unsortedtable_heap_unsafe (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_heap_unsafe OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ checkAnswer(sql("select * from unsortedtable_heap_unsafe where empno = 11"), sql("select * from origintable1 where empno = 11"))
+ checkAnswer(sql("select * from unsortedtable_heap_unsafe order by empno"), sql("select * from origintable1 order by empno"))
+ } finally {
+ defaultLoadingProperties
+ }
+ }
+
+ test("unsorted table creation, query and loading with heap and inmemory sort config") {
+ try {
+ setLoadingProperties("false", "false", "true")
+ sql("CREATE TABLE unsortedtable_heap_inmemory (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_heap_inmemory OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ checkAnswer(sql("select * from unsortedtable_heap_inmemory where empno = 11"), sql("select * from origintable1 where empno = 11"))
+ checkAnswer(sql("select * from unsortedtable_heap_inmemory order by empno"), sql("select * from origintable1 order by empno"))
+ } finally {
+ defaultLoadingProperties
+ }
+ }
+
+ test("unsorted table creation, query and data loading with offheap and safe sort config") {
+ try {
+ setLoadingProperties("true", "false", "false")
+ sql("CREATE TABLE unsortedtable_offheap_safe (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_offheap_safe OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ checkAnswer(sql("select * from unsortedtable_offheap_safe where empno = 11"), sql("select * from origintable1 where empno = 11"))
+ checkAnswer(sql("select * from unsortedtable_offheap_safe order by empno"), sql("select * from origintable1 order by empno"))
+ } finally {
+ defaultLoadingProperties
+ }
+ }
+
+ test("unsorted table creation, query and data loading with offheap and unsafe sort config") {
+ try {
+ setLoadingProperties("true", "true", "false")
+ sql("CREATE TABLE unsortedtable_offheap_unsafe (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_offheap_unsafe OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ checkAnswer(sql("select * from unsortedtable_offheap_unsafe where empno = 11"), sql("select * from origintable1 where empno = 11"))
+ checkAnswer(sql("select * from unsortedtable_offheap_unsafe order by empno"), sql("select * from origintable1 order by empno"))
+ } finally {
+ defaultLoadingProperties
+ }
+ }
+
+ test("unsorted table creation, query and data loading with offheap and inmemory sort config") {
+ try {
+ setLoadingProperties("true", "false", "true")
+ sql("CREATE TABLE unsortedtable_offheap_inmemory (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_offheap_inmemory OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno = 11"), sql("select * from origintable1 where empno = 11"))
+ checkAnswer(sql("select * from unsortedtable_offheap_inmemory order by empno"), sql("select * from origintable1 order by empno"))
+ } finally {
+ defaultLoadingProperties
+ }
+ }
+
+
+ override def afterAll = {
+ dropTable
+ }
+
+ def dropTable = {
+ sql("drop table if exists origintable1")
+ sql("drop table if exists origintable2")
+ sql("drop table if exists sorttable1")
+ sql("drop table if exists sorttable2")
+ sql("drop table if exists sorttable3")
+ sql("drop table if exists sorttable4_offheap_safe")
+ sql("drop table if exists sorttable4_offheap_unsafe")
+ sql("drop table if exists sorttable4_offheap_inmemory")
+ sql("drop table if exists sorttable4_heap_safe")
+ sql("drop table if exists sorttable4_heap_unsafe")
+ sql("drop table if exists sorttable4_heap_inmemory")
+ sql("drop table if exists sorttable5")
+ sql("drop table if exists sorttable6")
+ sql("drop table if exists unsortedtable_offheap_safe")
+ sql("drop table if exists unsortedtable_offheap_unsafe")
+ sql("drop table if exists unsortedtable_offheap_inmemory")
+ sql("drop table if exists unsortedtable_heap_safe")
+ sql("drop table if exists unsortedtable_heap_unsafe")
+ sql("drop table if exists unsortedtable_heap_inmemory")
+ }
+
+ def setLoadingProperties(offheap: String, unsafe: String, useBatch: String): Unit = {
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT, offheap)
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_UNSAFE_SORT, unsafe)
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.LOAD_USE_BATCH_SORT, useBatch)
+ }
+
+ def defaultLoadingProperties = {
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT, CarbonCommonConstants.ENABLE_OFFHEAP_SORT)
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_UNSAFE_SORT, CarbonCommonConstants.ENABLE_UNSAFE_SORT_DEFAULT)
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.LOAD_USE_BATCH_SORT, CarbonCommonConstants.LOAD_USE_BATCH_SORT_DEFAULT)
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index afc4a58..0df3b31 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -33,7 +33,7 @@ import org.apache.spark.sql.execution.command._
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.metadata.datatype.DataType
-import org.apache.carbondata.core.util.DataTypeUtil
+import org.apache.carbondata.core.util.{CarbonUtil, DataTypeUtil}
import org.apache.carbondata.processing.constants.LoggerAction
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
import org.apache.carbondata.spark.util.CommonUtil
@@ -240,21 +240,15 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
fields.zipWithIndex.foreach { case (field, index) =>
field.schemaOrdinal = index
}
- val (dims: Seq[Field], noDictionaryDims: Seq[String]) = extractDimColsAndNoDictionaryFields(
+ val (dims, msrs, noDictionaryDims, sortKeyDims) = extractDimAndMsrFields(
fields, tableProperties)
if (dims.isEmpty && !isAlterFlow) {
- throw new MalformedCarbonCommandException(s"Table ${
- dbName.getOrElse(
- CarbonCommonConstants.DATABASE_DEFAULT_NAME)
- }.$tableName"
- +
- " can not be created without key columns. Please " +
- "use DICTIONARY_INCLUDE or " +
- "DICTIONARY_EXCLUDE to set at least one key " +
- "column " +
- "if all specified columns are numeric types")
+ throw new MalformedCarbonCommandException(
+ s"Table ${dbName.getOrElse(CarbonCommonConstants.DATABASE_DEFAULT_NAME)}.$tableName " +
+ "can not be created without key columns. Please use DICTIONARY_INCLUDE or " +
+ "DICTIONARY_EXCLUDE to set at least one key column " +
+ "if all specified columns are numeric types")
}
- val msrs: Seq[Field] = extractMsrColsFromFields(fields, tableProperties)
// column properties
val colProps = extractColumnProperties(fields, tableProperties)
@@ -276,6 +270,7 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
tableProperties,
reorderDimensions(dims.map(f => normalizeType(f)).map(f => addParent(f))),
msrs.map(f => normalizeType(f)),
+ Option(sortKeyDims),
Option(noDictionaryDims),
Option(noInvertedIdxCols),
groupCols,
@@ -485,14 +480,50 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
* @param tableProperties
* @return
*/
- protected def extractDimColsAndNoDictionaryFields(fields: Seq[Field],
- tableProperties: Map[String, String]):
- (Seq[Field], Seq[String]) = {
+ protected def extractDimAndMsrFields(fields: Seq[Field],
+ tableProperties: Map[String, String]): (Seq[Field], Seq[Field], Seq[String], Seq[String]) = {
var dimFields: LinkedHashSet[Field] = LinkedHashSet[Field]()
+ var msrFields: Seq[Field] = Seq[Field]()
var dictExcludeCols: Array[String] = Array[String]()
var noDictionaryDims: Seq[String] = Seq[String]()
var dictIncludeCols: Seq[String] = Seq[String]()
+ // All columns in sortkey should be there in create table cols
+ val sortKeyOption = tableProperties.get(CarbonCommonConstants.SORT_COLUMNS)
+ var sortKeyDimsTmp: Seq[String] = Seq[String]()
+ val sortKeyString: String = if (sortKeyOption.isDefined) {
+ CarbonUtil.unquoteChar(sortKeyOption.get) trim
+ } else {
+ ""
+ }
+ if (!sortKeyString.isEmpty) {
+ val sortKey = sortKeyString.split(',').map(_.trim)
+ sortKey.foreach { column =>
+ if (!fields.exists(x => x.column.equalsIgnoreCase(column))) {
+ val errormsg = "sort_columns: " + column +
+ " does not exist in table. Please check create table statement."
+ throw new MalformedCarbonCommandException(errormsg)
+ } else {
+ val dataType = fields.find(x =>
+ x.column.equalsIgnoreCase(column)).get.dataType.get
+ if (isComplexDimDictionaryExclude(dataType)) {
+ val errormsg = "sort_columns is unsupported for complex datatype column: " + column
+ throw new MalformedCarbonCommandException(errormsg)
+ }
+ }
+ }
+
+ sortKey.foreach { dimension =>
+ if (!sortKeyDimsTmp.exists(dimension.equalsIgnoreCase(_))) {
+ fields.foreach { field =>
+ if (field.column.equalsIgnoreCase(dimension)) {
+ sortKeyDimsTmp :+= field.column
+ }
+ }
+ }
+ }
+ }
+
// All excluded cols should be there in create table cols
if (tableProperties.get(CarbonCommonConstants.DICTIONARY_EXCLUDE).isDefined) {
dictExcludeCols =
@@ -531,7 +562,7 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
}
}
- // include cols should contain exclude cols
+ // include cols should not contain exclude cols
dictExcludeCols.foreach { dicExcludeCol =>
if (dictIncludeCols.exists(x => x.equalsIgnoreCase(dicExcludeCol))) {
val errormsg = "DICTIONARY_EXCLUDE can not contain the same column: " + dicExcludeCol +
@@ -553,10 +584,30 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
dimFields += field
} else if (isDetectAsDimentionDatatype(field.dataType.get)) {
dimFields += field
+ } else if (sortKeyDimsTmp.exists(x => x.equalsIgnoreCase(field.column))) {
+ noDictionaryDims :+= field.column
+ dimFields += field
+ } else {
+ msrFields :+= field
}
}
- (dimFields.toSeq, noDictionaryDims)
+ var sortKeyDims = sortKeyDimsTmp
+ if (sortKeyOption.isEmpty) {
+ // if SORT_COLUMNS was not defined, add all dimension to SORT_COLUMNS.
+ dimFields.foreach { field =>
+ if (!isComplexDimDictionaryExclude(field.dataType.get)) {
+ sortKeyDims :+= field.column
+ }
+ }
+ }
+ if (sortKeyDims.isEmpty) {
+ // no SORT_COLUMNS
+ tableProperties.put(CarbonCommonConstants.SORT_COLUMNS, "")
+ } else {
+ tableProperties.put(CarbonCommonConstants.SORT_COLUMNS, sortKeyDims.mkString(","))
+ }
+ (dimFields.toSeq, msrFields, noDictionaryDims, sortKeyDims)
}
/**
@@ -603,44 +654,6 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
}
/**
- * Extract the Measure Cols fields. By default all non string cols will be measures.
- *
- * @param fields
- * @param tableProperties
- * @return
- */
- protected def extractMsrColsFromFields(fields: Seq[Field],
- tableProperties: Map[String, String]): Seq[Field] = {
- var msrFields: Seq[Field] = Seq[Field]()
- var dictIncludedCols: Array[String] = Array[String]()
- var dictExcludedCols: Array[String] = Array[String]()
-
- // get all included cols
- if (tableProperties.get(CarbonCommonConstants.DICTIONARY_INCLUDE).isDefined) {
- dictIncludedCols =
- tableProperties.get(CarbonCommonConstants.DICTIONARY_INCLUDE).get.split(',').map(_.trim)
- }
-
- // get all excluded cols
- if (tableProperties.get(CarbonCommonConstants.DICTIONARY_EXCLUDE).isDefined) {
- dictExcludedCols =
- tableProperties.get(CarbonCommonConstants.DICTIONARY_EXCLUDE).get.split(',').map(_.trim)
- }
-
- // by default consider all non string cols as msrs. consider all include/ exclude cols as dims
- fields.foreach(field => {
- if (!isDetectAsDimentionDatatype(field.dataType.get)) {
- if (!dictIncludedCols.exists(x => x.equalsIgnoreCase(field.column)) &&
- !dictExcludedCols.exists(x => x.equalsIgnoreCase(field.column))) {
- msrFields :+= field
- }
- }
- })
-
- msrFields
- }
-
- /**
* Extract the DbName and table name.
*
* @param tableNameParts
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index e03cbe5..187512d 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -54,6 +54,7 @@ case class TableModel(
tableProperties: Map[String, String],
dimCols: Seq[Field],
msrCols: Seq[Field],
+ sortKeyDims: Option[Seq[String]],
highcardinalitydims: Option[Seq[String]],
noInvertedIdxCols: Option[Seq[String]],
columnGroups: Seq[String],
@@ -348,6 +349,7 @@ class TableNewProcessor(cm: TableModel) {
columnSchema.setPrecision(precision)
columnSchema.setScale(scale)
columnSchema.setSchemaOrdinal(schemaOrdinal)
+ columnSchema.setSortColumn(false)
// TODO: Need to fill RowGroupID, converted type
// & Number of Children after DDL finalization
columnSchema
@@ -358,7 +360,11 @@ class TableNewProcessor(cm: TableModel) {
val LOGGER = LogServiceFactory.getLogService(TableNewProcessor.getClass.getName)
var allColumns = Seq[ColumnSchema]()
var index = 0
- cm.dimCols.foreach(field => {
+ var measureCount = 0
+
+ // Sort columns should be at the begin of all columns
+ cm.sortKeyDims.get.foreach { keyDim =>
+ val field = cm.dimCols.find(keyDim equals _.column).get
val encoders = new java.util.ArrayList[Encoding]()
encoders.add(Encoding.DICTIONARY)
val columnSchema: ColumnSchema = getColumnSchema(
@@ -372,11 +378,33 @@ class TableNewProcessor(cm: TableModel) {
field.precision,
field.scale,
field.schemaOrdinal)
- allColumns ++= Seq(columnSchema)
+ columnSchema.setSortColumn(true)
+ allColumns :+= columnSchema
index = index + 1
- if (field.children.isDefined && field.children.get != null) {
- columnSchema.setNumberOfChild(field.children.get.size)
- allColumns ++= getAllChildren(field.children)
+ }
+
+ cm.dimCols.foreach(field => {
+ val sortField = cm.sortKeyDims.get.find(field.column equals _)
+ if (sortField.isEmpty) {
+ val encoders = new java.util.ArrayList[Encoding]()
+ encoders.add(Encoding.DICTIONARY)
+ val columnSchema: ColumnSchema = getColumnSchema(
+ DataTypeConverterUtil.convertToCarbonType(field.dataType.getOrElse("")),
+ field.name.getOrElse(field.column),
+ index,
+ isCol = true,
+ encoders,
+ isDimensionCol = true,
+ -1,
+ field.precision,
+ field.scale,
+ field.schemaOrdinal)
+ allColumns :+= columnSchema
+ index = index + 1
+ if (field.children.isDefined && field.children.get != null) {
+ columnSchema.setNumberOfChild(field.children.get.size)
+ allColumns ++= getAllChildren(field.children)
+ }
}
})
@@ -393,10 +421,9 @@ class TableNewProcessor(cm: TableModel) {
field.precision,
field.scale,
field.schemaOrdinal)
- val measureCol = columnSchema
-
- allColumns ++= Seq(measureCol)
+ allColumns :+= columnSchema
index = index + 1
+ measureCount += 1
})
// Check if there is any duplicate measures or dimensions.
@@ -417,22 +444,6 @@ class TableNewProcessor(cm: TableModel) {
updateColumnGroupsInFields(cm.columnGroups, allColumns)
- var newOrderedDims = scala.collection.mutable.ListBuffer[ColumnSchema]()
- val complexDims = scala.collection.mutable.ListBuffer[ColumnSchema]()
- val measures = scala.collection.mutable.ListBuffer[ColumnSchema]()
- for (column <- allColumns) {
- if (highCardinalityDims.contains(column.getColumnName)) {
- newOrderedDims += column
- } else if (column.isComplex) {
- complexDims += column
- } else if (column.isDimensionColumn) {
- newOrderedDims += column
- } else {
- measures += column
- }
-
- }
-
// Setting the boolean value of useInvertedIndex in column schema
val noInvertedIndexCols = cm.noInvertedIdxCols.getOrElse(Seq())
for (column <- allColumns) {
@@ -447,7 +458,7 @@ class TableNewProcessor(cm: TableModel) {
}
// Adding dummy measure if no measure is provided
- if (measures.size < 1) {
+ if (measureCount == 0) {
val encoders = new java.util.ArrayList[Encoding]()
val columnSchema: ColumnSchema = getColumnSchema(DataType.DOUBLE,
CarbonCommonConstants.DEFAULT_INVISIBLE_DUMMY_MEASURE,
@@ -457,13 +468,10 @@ class TableNewProcessor(cm: TableModel) {
false,
-1, 0, 0, schemaOrdinal = -1)
columnSchema.setInvisible(true)
- val measureColumn = columnSchema
- measures += measureColumn
- allColumns = allColumns ++ measures
+ allColumns :+= columnSchema
}
val columnValidator = CarbonSparkFactory.getCarbonColumnValidator()
columnValidator.validateColumns(allColumns)
- newOrderedDims = newOrderedDims ++ complexDims ++ measures
val tableInfo = new TableInfo()
val tableSchema = new TableSchema()
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/integration/spark/src/test/scala/org/apache/spark/sql/TestCarbonSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/spark/sql/TestCarbonSqlParser.scala b/integration/spark/src/test/scala/org/apache/spark/sql/TestCarbonSqlParser.scala
index 8588868..b8f0a7c 100644
--- a/integration/spark/src/test/scala/org/apache/spark/sql/TestCarbonSqlParser.scala
+++ b/integration/spark/src/test/scala/org/apache/spark/sql/TestCarbonSqlParser.scala
@@ -32,23 +32,15 @@ private class TestCarbonSqlParserStub extends CarbonSqlParser {
def updateColumnGroupsInFieldTest(fields: Seq[Field], tableProperties: Map[String, String]): Seq[String] = {
- var (dims: Seq[Field], noDictionaryDims: Seq[String]) = extractDimColsAndNoDictionaryFields(
- fields, tableProperties)
- val msrs: Seq[Field] = extractMsrColsFromFields(fields, tableProperties)
+ var (dims, msrs, noDictionaryDims, sortkey) = extractDimAndMsrFields(fields, tableProperties)
updateColumnGroupsInField(tableProperties,
noDictionaryDims, msrs, dims)
}
- def extractDimColsAndNoDictionaryFieldsTest(fields: Seq[Field], tableProperties: Map[String, String]): (Seq[Field],
- Seq[String]) = {
-
- extractDimColsAndNoDictionaryFields(fields, tableProperties)
- }
-
- def extractMsrColsFromFieldsTest(fields: Seq[Field], tableProperties: Map[String, String]): (Seq[Field]) = {
-
- extractMsrColsFromFields(fields, tableProperties)
+ def extractDimAndMsrFieldsTest(fields: Seq[Field],
+ tableProperties: Map[String, String]): (Seq[Field], Seq[Field], Seq[String], Seq[String]) = {
+ extractDimAndMsrFields(fields, tableProperties)
}
@@ -199,7 +191,7 @@ class TestCarbonSqlParser extends QueryTest {
val fields: Seq[Field] = loadAllFields
val stub = new TestCarbonSqlParserStub()
- val (dimCols, noDictionary) = stub.extractDimColsAndNoDictionaryFieldsTest(fields, tableProperties)
+ val (dimCols, _, noDictionary, _) = stub.extractDimAndMsrFieldsTest(fields, tableProperties)
// testing col
@@ -219,9 +211,7 @@ class TestCarbonSqlParser extends QueryTest {
val tableProperties = Map(CarbonCommonConstants.DICTIONARY_EXCLUDE -> "col1")
val fields: Seq[Field] = loadAllFields
val stub = new TestCarbonSqlParserStub()
- val (dimCols, noDictionary) = stub
- .extractDimColsAndNoDictionaryFieldsTest(fields, tableProperties)
- val msrCols = stub.extractMsrColsFromFieldsTest(fields, tableProperties)
+ val (dimCols, msrCols, noDictionary, _) = stub.extractDimAndMsrFieldsTest(fields, tableProperties)
//below fields should be available in dimensions list
assert(dimCols.size == 7)
@@ -242,9 +232,7 @@ class TestCarbonSqlParser extends QueryTest {
val tableProperties = Map(CarbonCommonConstants.DICTIONARY_INCLUDE -> "col1")
val fields: Seq[Field] = loadAllFields
val stub = new TestCarbonSqlParserStub()
- val (dimCols, noDictionary) = stub
- .extractDimColsAndNoDictionaryFieldsTest(fields, tableProperties)
- val msrCols = stub.extractMsrColsFromFieldsTest(fields, tableProperties)
+ val (dimCols, msrCols, noDictionary, _) = stub.extractDimAndMsrFieldsTest(fields, tableProperties)
//below dimension fields should be available in dimensions list
assert(dimCols.size == 7)
@@ -264,9 +252,8 @@ class TestCarbonSqlParser extends QueryTest {
val tableProperties = Map(CarbonCommonConstants.DICTIONARY_EXCLUDE -> "col1", CarbonCommonConstants.DICTIONARY_INCLUDE -> "col4")
val fields: Seq[Field] = loadAllFields
val stub = new TestCarbonSqlParserStub()
- val (dimCols, noDictionary) = stub
- .extractDimColsAndNoDictionaryFieldsTest(fields, tableProperties)
- val msrCols = stub.extractMsrColsFromFieldsTest(fields, tableProperties)
+ val (dimCols, msrCols, noDictionary, _) = stub.extractDimAndMsrFieldsTest(fields,
+ tableProperties)
//below dimension fields should be available in dimensions list
assert(dimCols.size == 8)
@@ -287,9 +274,7 @@ class TestCarbonSqlParser extends QueryTest {
val tableProperties = Map(CarbonCommonConstants.DICTIONARY_EXCLUDE -> "col3", CarbonCommonConstants.DICTIONARY_INCLUDE -> "col2")
val fields: Seq[Field] = loadAllFields
val stub = new TestCarbonSqlParserStub()
- val (dimCols, noDictionary) = stub
- .extractDimColsAndNoDictionaryFieldsTest(fields, tableProperties)
- val msrCols = stub.extractMsrColsFromFieldsTest(fields, tableProperties)
+ val (dimCols, msrCols, noDictionary, _) = stub.extractDimAndMsrFieldsTest(fields, tableProperties)
//below dimension fields should be available in dimensions list
assert(dimCols.size == 7)
@@ -310,9 +295,7 @@ class TestCarbonSqlParser extends QueryTest {
val tableProperties = Map(CarbonCommonConstants.DICTIONARY_EXCLUDE -> "col1", CarbonCommonConstants.DICTIONARY_INCLUDE -> "col2")
val fields: Seq[Field] = loadAllFields
val stub = new TestCarbonSqlParserStub()
- val (dimCols, noDictionary) = stub
- .extractDimColsAndNoDictionaryFieldsTest(fields, tableProperties)
- val msrCols = stub.extractMsrColsFromFieldsTest(fields, tableProperties)
+ val (dimCols, msrCols, noDictionary, _) = stub.extractDimAndMsrFieldsTest(fields, tableProperties)
//below dimension fields should be available in dimensions list
assert(dimCols.size == 7)
@@ -333,9 +316,7 @@ class TestCarbonSqlParser extends QueryTest {
val tableProperties = Map(CarbonCommonConstants.DICTIONARY_EXCLUDE -> "col2", CarbonCommonConstants.DICTIONARY_INCLUDE -> "col1")
val fields: Seq[Field] = loadAllFields
val stub = new TestCarbonSqlParserStub()
- val (dimCols, noDictionary) = stub
- .extractDimColsAndNoDictionaryFieldsTest(fields, tableProperties)
- val msrCols = stub.extractMsrColsFromFieldsTest(fields, tableProperties)
+ val (dimCols, msrCols, noDictionary, _) = stub.extractDimAndMsrFieldsTest(fields, tableProperties)
//below dimension fields should be available in dimensions list
assert(dimCols.size == 7)
@@ -358,9 +339,7 @@ class TestCarbonSqlParser extends QueryTest {
)
val fields: Seq[Field] = loadAllFields
val stub = new TestCarbonSqlParserStub()
- val (dimCols, noDictionary) = stub
- .extractDimColsAndNoDictionaryFieldsTest(fields, tableProperties)
- val msrCols = stub.extractMsrColsFromFieldsTest(fields, tableProperties)
+ val (dimCols, msrCols, noDictionary, _) = stub.extractDimAndMsrFieldsTest(fields, tableProperties)
//below dimension fields should be available in dimensions list
assert(dimCols.size == 8)
@@ -382,9 +361,7 @@ class TestCarbonSqlParser extends QueryTest {
val tableProperties = Map(CarbonCommonConstants.DICTIONARY_EXCLUDE-> "col2", CarbonCommonConstants.DICTIONARY_INCLUDE -> "col3")
val fields: Seq[Field] = loadAllFields
val stub = new TestCarbonSqlParserStub()
- val (dimCols, noDictionary) = stub
- .extractDimColsAndNoDictionaryFieldsTest(fields, tableProperties)
- val msrCols = stub.extractMsrColsFromFieldsTest(fields, tableProperties)
+ val (dimCols, msrCols, noDictionary, _) = stub.extractDimAndMsrFieldsTest(fields, tableProperties)
//below dimension fields should be available in dimensions list
assert(dimCols.size == 7)
@@ -402,10 +379,11 @@ class TestCarbonSqlParser extends QueryTest {
// Testing the extracting of measures
test("Test-extractMsrColsFromFields") {
- val tableProperties = Map(CarbonCommonConstants.DICTIONARY_EXCLUDE -> "col2", CarbonCommonConstants.DICTIONARY_INCLUDE -> "col1")
+ val tableProperties = Map(CarbonCommonConstants.DICTIONARY_EXCLUDE -> "col2",
+ CarbonCommonConstants.DICTIONARY_INCLUDE -> "col1")
val fields: Seq[Field] = loadAllFields
val stub = new TestCarbonSqlParserStub()
- val msrCols = stub.extractMsrColsFromFieldsTest(fields, tableProperties)
+ val (_, msrCols, _, _) = stub.extractDimAndMsrFieldsTest(fields, tableProperties)
// testing col
assert(msrCols.lift(0).get.column.equalsIgnoreCase("col4"))
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/integration/spark2/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java b/integration/spark2/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
index b848543..b4b462a 100644
--- a/integration/spark2/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
+++ b/integration/spark2/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
@@ -28,18 +28,25 @@ import org.apache.spark.sql.catalyst.expressions.GenericInternalRow;
public class SparkRowReadSupportImpl extends DictionaryDecodeReadSupport<InternalRow> {
+ boolean[] isMeasure;
+
@Override public void initialize(CarbonColumn[] carbonColumns,
AbsoluteTableIdentifier absoluteTableIdentifier) throws IOException {
- super.initialize(carbonColumns, absoluteTableIdentifier);
//can initialize and generate schema here.
+ isMeasure = new boolean[carbonColumns.length];
+ dataTypes = new DataType[carbonColumns.length];
+ for (int i = 0; i < carbonColumns.length; i++) {
+ isMeasure[i] = !carbonColumns[i].isDimesion();
+ dataTypes[i] = carbonColumns[i].getDataType();
+ }
}
@Override public InternalRow readRow(Object[] data) {
- for (int i = 0; i < dictionaries.length; i++) {
+ for (int i = 0; i < isMeasure.length; i++) {
if (data[i] == null) {
continue;
}
- if (dictionaries[i] == null) {
+ if (isMeasure[i]) {
if (dataTypes[i].equals(DataType.INT)) {
data[i] = ((Long)(data[i])).intValue();
} else if (dataTypes[i].equals(DataType.SHORT)) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapper.java
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapper.java b/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapper.java
index f94c0b2..ad5e01f 100644
--- a/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapper.java
+++ b/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapper.java
@@ -20,6 +20,7 @@ package org.apache.carbondata.spark.vectorreader;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
import org.apache.spark.sql.execution.vectorized.ColumnVector;
+import org.apache.spark.sql.types.DataType;
import org.apache.spark.sql.types.Decimal;
class ColumnarVectorWrapper implements CarbonColumnVector {
@@ -30,6 +31,14 @@ class ColumnarVectorWrapper implements CarbonColumnVector {
this.columnVector = columnVector;
}
+ @Override public void putBoolean(int rowId, boolean value) {
+ columnVector.putBoolean(rowId, value);
+ }
+
+ @Override public void putFloat(int rowId, float value) {
+ columnVector.putFloat(rowId, value);
+ }
+
@Override public void putShort(int rowId, short value) {
columnVector.putShort(rowId, value);
}
@@ -110,4 +119,8 @@ class ColumnarVectorWrapper implements CarbonColumnVector {
@Override public void reset() {
// columnVector.reset();
}
+
+ @Override public DataType getType() {
+ return columnVector.dataType();
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
index c4f9fc5..068f852 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
@@ -25,7 +25,7 @@ import org.apache.spark.sql.execution.command._
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.metadata.datatype.DataType
-import org.apache.carbondata.core.util.DataTypeUtil
+import org.apache.carbondata.core.util.{CarbonUtil, DataTypeUtil}
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
import org.apache.carbondata.spark.util.CommonUtil
@@ -51,18 +51,54 @@ object TableCreator {
dimensionTypes.exists(dimensionType => dimensionType.equalsIgnoreCase(dimensionDatatype))
}
- protected def extractDimColsAndNoDictionaryFields(fields: Seq[Field],
- tableProperties: Map[String, String]):
- (Seq[Field], Seq[String]) = {
+ protected def extractDimAndMsrFields(fields: Seq[Field],
+ tableProperties: Map[String, String]): (Seq[Field], Seq[Field], Seq[String], Seq[String]) = {
var dimFields: LinkedHashSet[Field] = LinkedHashSet[Field]()
+ var msrFields: Seq[Field] = Seq[Field]()
var dictExcludeCols: Array[String] = Array[String]()
var noDictionaryDims: Seq[String] = Seq[String]()
var dictIncludeCols: Seq[String] = Seq[String]()
+ // All columns in sortkey should be there in create table cols
+ val sortKeyOption = tableProperties.get(CarbonCommonConstants.SORT_COLUMNS)
+ var sortKeyDimsTmp: Seq[String] = Seq[String]()
+ val sortKeyString: String = if (sortKeyOption.isDefined) {
+ CarbonUtil.unquoteChar(sortKeyOption.get) trim
+ } else {
+ ""
+ }
+ if (!sortKeyString.isEmpty) {
+ val sortKey = sortKeyString.split(',').map(_.trim)
+ sortKey.foreach { column =>
+ if (!fields.exists(x => x.column.equalsIgnoreCase(column))) {
+ val errormsg = "sort_columns: " + column +
+ " does not exist in table. Please check create table statement."
+ throw new MalformedCarbonCommandException(errormsg)
+ } else {
+ val dataType = fields.find(x =>
+ x.column.equalsIgnoreCase(column)).get.dataType.get
+ if (isComplexDimDictionaryExclude(dataType)) {
+ val errormsg = "sort_columns is unsupported for complex datatype column: " + column
+ throw new MalformedCarbonCommandException(errormsg)
+ }
+ }
+ }
+
+ sortKey.foreach { dimension =>
+ if (!sortKeyDimsTmp.exists(dimension.equalsIgnoreCase(_))) {
+ fields.foreach { field =>
+ if (field.column.equalsIgnoreCase(dimension)) {
+ sortKeyDimsTmp :+= field.column
+ }
+ }
+ }
+ }
+ }
+
// All excluded cols should be there in create table cols
if (tableProperties.get(CarbonCommonConstants.DICTIONARY_EXCLUDE).isDefined) {
dictExcludeCols =
- tableProperties(CarbonCommonConstants.DICTIONARY_EXCLUDE).split(',').map(_.trim)
+ tableProperties.get(CarbonCommonConstants.DICTIONARY_EXCLUDE).get.split(',').map(_.trim)
dictExcludeCols
.foreach { dictExcludeCol =>
if (!fields.exists(field => field.column.equalsIgnoreCase(dictExcludeCol))) {
@@ -97,7 +133,7 @@ object TableCreator {
}
}
- // include cols should contain exclude cols
+ // include cols should not contain exclude cols
dictExcludeCols.foreach { dicExcludeCol =>
if (dictIncludeCols.exists(col => col.equalsIgnoreCase(dicExcludeCol))) {
val errormsg = "DICTIONARY_EXCLUDE can not contain the same column: " + dicExcludeCol +
@@ -108,11 +144,10 @@ object TableCreator {
// by default consider all String cols as dims and if any dictionary exclude is present then
// add it to noDictionaryDims list. consider all dictionary excludes/include cols as dims
- fields.foreach(field => {
-
+ fields.foreach { field =>
if (dictExcludeCols.toSeq.exists(col => col.equalsIgnoreCase(field.column))) {
- if (DataTypeUtil.getDataType(field.dataType.get.toUpperCase()) != DataType.TIMESTAMP &&
- DataTypeUtil.getDataType(field.dataType.get.toUpperCase()) != DataType.DATE) {
+ val dataType = DataTypeUtil.getDataType(field.dataType.get.toUpperCase())
+ if (dataType != DataType.TIMESTAMP && dataType != DataType.DATE) {
noDictionaryDims :+= field.column
}
dimFields += field
@@ -120,49 +155,30 @@ object TableCreator {
dimFields += field
} else if (isDetectAsDimentionDatatype(field.dataType.get)) {
dimFields += field
+ } else if (sortKeyDimsTmp.exists(x => x.equalsIgnoreCase(field.column))) {
+ noDictionaryDims :+= field.column
+ dimFields += field
+ } else {
+ msrFields :+= field
}
}
- )
-
- (dimFields.toSeq, noDictionaryDims)
- }
-
- /**
- * Extract the Measure Cols fields. By default all non string cols will be measures.
- *
- * @param fields
- * @param tableProperties
- * @return
- */
- protected def extractMsrColsFromFields(fields: Seq[Field],
- tableProperties: Map[String, String]): Seq[Field] = {
- var msrFields: Seq[Field] = Seq[Field]()
- var dictIncludedCols: Array[String] = Array[String]()
- var dictExcludedCols: Array[String] = Array[String]()
-
- // get all included cols
- if (tableProperties.get(CarbonCommonConstants.DICTIONARY_INCLUDE).isDefined) {
- dictIncludedCols =
- tableProperties(CarbonCommonConstants.DICTIONARY_INCLUDE).split(',').map(_.trim)
- }
-
- // get all excluded cols
- if (tableProperties.get(CarbonCommonConstants.DICTIONARY_EXCLUDE).isDefined) {
- dictExcludedCols =
- tableProperties(CarbonCommonConstants.DICTIONARY_EXCLUDE).split(',').map(_.trim)
- }
- // by default consider all non string cols as msrs. consider all include/ exclude cols as dims
- fields.foreach(field => {
- if (!isDetectAsDimentionDatatype(field.dataType.get)) {
- if (!dictIncludedCols.exists(col => col.equalsIgnoreCase(field.column)) &&
- !dictExcludedCols.exists(col => col.equalsIgnoreCase(field.column))) {
- msrFields :+= field
+ var sortKeyDims = sortKeyDimsTmp
+ if (sortKeyOption.isEmpty) {
+ // if SORT_COLUMNS was not defined, add all dimension to SORT_COLUMNS.
+ dimFields.foreach { field =>
+ if (!isComplexDimDictionaryExclude(field.dataType.get)) {
+ sortKeyDims :+= field.column
}
}
- })
-
- msrFields
+ }
+ if (sortKeyDims.isEmpty) {
+ // no SORT_COLUMNS
+ tableProperties.put(CarbonCommonConstants.SORT_COLUMNS, "")
+ } else {
+ tableProperties.put(CarbonCommonConstants.SORT_COLUMNS, sortKeyDims.mkString(","))
+ }
+ (dimFields.toSeq, msrFields, noDictionaryDims, sortKeyDims)
}
def getKey(parentColumnName: Option[String],
@@ -440,27 +456,24 @@ object TableCreator {
}
def prepareTableModel(ifNotExistPresent: Boolean, dbName: Option[String]
- , tableName: String, fields: Seq[Field],
- partitionCols: Seq[PartitionerField],
- bucketFields: Option[BucketFields],
- tableProperties: Map[String, String]): TableModel
+ , tableName: String, fields: Seq[Field],
+ partitionCols: Seq[PartitionerField],
+ bucketFields: Option[BucketFields],
+ tableProperties: Map[String, String]): TableModel
= {
- val (dims: Seq[Field], noDictionaryDims: Seq[String]) = extractDimColsAndNoDictionaryFields(
+ fields.zipWithIndex.foreach { x =>
+ x._1.schemaOrdinal = x._2
+ }
+ val (dims, msrs, noDictionaryDims, sortKeyDims) = extractDimAndMsrFields(
fields, tableProperties)
if (dims.isEmpty) {
- throw new MalformedCarbonCommandException(s"Table ${
- dbName.getOrElse(
- CarbonCommonConstants.DATABASE_DEFAULT_NAME)
- }.$tableName"
- +
- " can not be created without key columns. Please " +
- "use DICTIONARY_INCLUDE or " +
- "DICTIONARY_EXCLUDE to set at least one key " +
- "column " +
+ throw new MalformedCarbonCommandException(
+ s"Table ${dbName.getOrElse(CarbonCommonConstants.DATABASE_DEFAULT_NAME)}.$tableName " +
+ "can not be created without key columns. Please use DICTIONARY_INCLUDE or " +
+ "DICTIONARY_EXCLUDE to set at least one key column " +
"if all specified columns are numeric types")
}
- val msrs: Seq[Field] = extractMsrColsFromFields(fields, tableProperties)
// column properties
val colProps = extractColumnProperties(fields, tableProperties)
@@ -474,18 +487,20 @@ object TableCreator {
// validate the tableBlockSize from table properties
CommonUtil.validateTableBlockSize(tableProperties)
- TableModel(ifNotExistPresent,
+ TableModel(
+ ifNotExistPresent,
dbName.getOrElse(CarbonCommonConstants.DATABASE_DEFAULT_NAME),
dbName,
tableName,
tableProperties,
reorderDimensions(dims.map(f => normalizeType(f)).map(f => addParent(f))),
msrs.map(f => normalizeType(f)),
+ Option(sortKeyDims),
Option(noDictionaryDims),
Option(noInvertedIdxCols),
groupCols,
Some(colProps),
- bucketFields)
+ bucketFields: Option[BucketFields])
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
index a76cc0a..a8f9520 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
@@ -359,10 +359,18 @@ public class CompactionResultSortProcessor extends AbstractResultProcessor {
private void initializeFinalThreadMergerForMergeSort() {
String sortTempFileLocation = tempStoreLocation + CarbonCommonConstants.FILE_SEPARATOR
+ CarbonCommonConstants.SORT_TEMP_FILE_LOCATION;
+ boolean[] noDictionarySortColumnMapping = null;
+ if (noDictionaryColMapping.length == this.segmentProperties.getNumberOfSortColumns()) {
+ noDictionarySortColumnMapping = noDictionaryColMapping;
+ } else {
+ noDictionarySortColumnMapping = new boolean[this.segmentProperties.getNumberOfSortColumns()];
+ System.arraycopy(noDictionaryColMapping, 0,
+ noDictionarySortColumnMapping, 0, noDictionarySortColumnMapping.length);
+ }
finalMerger =
new SingleThreadFinalSortFilesMerger(sortTempFileLocation, tableName, dimensionColumnCount,
segmentProperties.getComplexDimensions().size(), measureCount, noDictionaryCount,
- aggType, noDictionaryColMapping);
+ aggType, noDictionaryColMapping, noDictionarySortColumnMapping);
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java b/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java
index b7aa32c..ce85227 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java
@@ -183,6 +183,19 @@ public class RowResultMergerProcessor extends AbstractResultProcessor {
* Comparator class for comparing 2 raw row result.
*/
private class CarbonMdkeyComparator implements Comparator<RawResultIterator> {
+ int[] columnValueSizes = segprop.getEachDimColumnValueSize();
+ public CarbonMdkeyComparator() {
+ initSortColumns();
+ }
+
+ private void initSortColumns() {
+ int numberOfSortColumns = segprop.getNumberOfSortColumns();
+ if (numberOfSortColumns != columnValueSizes.length) {
+ int[] sortColumnValueSizes = new int[numberOfSortColumns];
+ System.arraycopy(columnValueSizes, 0, sortColumnValueSizes, 0, numberOfSortColumns);
+ this.columnValueSizes = sortColumnValueSizes;
+ }
+ }
@Override public int compare(RawResultIterator o1, RawResultIterator o2) {
@@ -200,7 +213,6 @@ public class RowResultMergerProcessor extends AbstractResultProcessor {
ByteArrayWrapper key1 = (ByteArrayWrapper) row1[0];
ByteArrayWrapper key2 = (ByteArrayWrapper) row2[0];
int compareResult = 0;
- int[] columnValueSizes = segprop.getEachDimColumnValueSize();
int dictionaryKeyOffset = 0;
byte[] dimCols1 = key1.getDictionaryKey();
byte[] dimCols2 = key2.getDictionaryKey();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/processing/src/main/java/org/apache/carbondata/processing/newflow/CarbonDataLoadConfiguration.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/CarbonDataLoadConfiguration.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/CarbonDataLoadConfiguration.java
index 12be777..e1eb071 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/CarbonDataLoadConfiguration.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/CarbonDataLoadConfiguration.java
@@ -80,6 +80,10 @@ public class CarbonDataLoadConfiguration {
private DictionaryCardinalityFinder cardinalityFinder;
+ private int numberOfSortColumns;
+
+ private int numberOfNoDictSortColumns;
+
public CarbonDataLoadConfiguration() {
}
@@ -125,6 +129,26 @@ public class CarbonDataLoadConfiguration {
return dimCount;
}
+ public void setNumberOfSortColumns(int numberOfSortColumns) {
+ this.numberOfSortColumns = numberOfSortColumns;
+ }
+
+ public int getNumberOfSortColumns() {
+ return this.numberOfSortColumns;
+ }
+
+ public boolean isSortTable() {
+ return this.numberOfSortColumns > 0;
+ }
+
+ public void setNumberOfNoDictSortColumns(int numberOfNoDictSortColumns) {
+ this.numberOfNoDictSortColumns = numberOfNoDictSortColumns;
+ }
+
+ public int getNumberOfNoDictSortColumns() {
+ return this.numberOfNoDictSortColumns;
+ }
+
public int getComplexDimensionCount() {
int dimCount = 0;
for (int i = 0; i < dataFields.length; i++) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
index 8865518..11e5db1 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
@@ -40,6 +40,7 @@ import org.apache.carbondata.processing.newflow.steps.DataConverterProcessorWith
import org.apache.carbondata.processing.newflow.steps.DataWriterBatchProcessorStepImpl;
import org.apache.carbondata.processing.newflow.steps.DataWriterProcessorStepImpl;
import org.apache.carbondata.processing.newflow.steps.InputProcessorStepImpl;
+import org.apache.carbondata.processing.newflow.steps.NoSortProcessorStepImpl;
import org.apache.carbondata.processing.newflow.steps.SortProcessorStepImpl;
import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
@@ -76,9 +77,10 @@ public final class DataLoadProcessBuilder {
// data types and configurations.
AbstractDataLoadProcessorStep converterProcessorStep =
new DataConverterProcessorStepImpl(configuration, inputProcessorStep);
- // 3. Sorts the data which are part of key (all dimensions except complex types)
- AbstractDataLoadProcessorStep sortProcessorStep =
- new SortProcessorStepImpl(configuration, converterProcessorStep);
+ // 3. Sorts the data by SortColumn or not
+ AbstractDataLoadProcessorStep sortProcessorStep = configuration.isSortTable() ?
+ new SortProcessorStepImpl(configuration, converterProcessorStep) :
+ new NoSortProcessorStepImpl(configuration, converterProcessorStep);
// 4. Writes the sorted data in carbondata format.
AbstractDataLoadProcessorStep writerProcessorStep =
new DataWriterProcessorStepImpl(configuration, sortProcessorStep);
@@ -94,9 +96,10 @@ public final class DataLoadProcessBuilder {
// data types and configurations.
AbstractDataLoadProcessorStep converterProcessorStep =
new DataConverterProcessorStepImpl(configuration, inputProcessorStep);
- // 3. Sorts the data which are part of key (all dimensions except complex types)
- AbstractDataLoadProcessorStep sortProcessorStep =
- new SortProcessorStepImpl(configuration, converterProcessorStep);
+ // 3. Sorts the data by SortColumn or not
+ AbstractDataLoadProcessorStep sortProcessorStep = configuration.isSortTable() ?
+ new SortProcessorStepImpl(configuration, converterProcessorStep) :
+ new NoSortProcessorStepImpl(configuration, converterProcessorStep);
// 4. Writes the sorted data in carbondata format.
AbstractDataLoadProcessorStep writerProcessorStep =
new DataWriterBatchProcessorStepImpl(configuration, sortProcessorStep);
@@ -112,9 +115,10 @@ public final class DataLoadProcessBuilder {
// data types and configurations.
AbstractDataLoadProcessorStep converterProcessorStep =
new DataConverterProcessorWithBucketingStepImpl(configuration, inputProcessorStep);
- // 3. Sorts the data which are part of key (all dimensions except complex types)
- AbstractDataLoadProcessorStep sortProcessorStep =
- new SortProcessorStepImpl(configuration, converterProcessorStep);
+ // 3. Sorts the data by SortColumn or not
+ AbstractDataLoadProcessorStep sortProcessorStep = configuration.isSortTable() ?
+ new SortProcessorStepImpl(configuration, converterProcessorStep) :
+ new NoSortProcessorStepImpl(configuration, converterProcessorStep);
// 4. Writes the sorted data in carbondata format.
AbstractDataLoadProcessorStep writerProcessorStep =
new DataWriterProcessorStepImpl(configuration, sortProcessorStep);
@@ -193,6 +197,8 @@ public final class DataLoadProcessBuilder {
configuration.setDictionaryServerHost(loadModel.getDictionaryServerHost());
configuration.setDictionaryServerPort(loadModel.getDictionaryServerPort());
configuration.setPreFetch(loadModel.isPreFetch());
+ configuration.setNumberOfSortColumns(carbonTable.getNumberOfSortColumns());
+ configuration.setNumberOfNoDictSortColumns(carbonTable.getNumberOfNoDictSortColumns());
return configuration;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/NonDictionaryFieldConverterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/NonDictionaryFieldConverterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/NonDictionaryFieldConverterImpl.java
index 9e4b50d..3accb0b 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/NonDictionaryFieldConverterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/NonDictionaryFieldConverterImpl.java
@@ -16,8 +16,6 @@
*/
package org.apache.carbondata.processing.newflow.converter.impl;
-import java.nio.charset.Charset;
-
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
@@ -48,23 +46,24 @@ public class NonDictionaryFieldConverterImpl implements FieldConverter {
this.isEmptyBadRecord = isEmptyBadRecord;
}
- @Override
- public void convert(CarbonRow row, BadRecordLogHolder logHolder) {
+ @Override public void convert(CarbonRow row, BadRecordLogHolder logHolder) {
String dimensionValue = row.getString(index);
if (dimensionValue == null || dimensionValue.equals(nullformat)) {
- dimensionValue = CarbonCommonConstants.MEMBER_DEFAULT_VAL;
- }
- if (dataType != DataType.STRING) {
- if (null == DataTypeUtil.normalizeIntAndLongValues(dimensionValue, dataType)) {
- if ((dimensionValue.length() > 0) || (dimensionValue.length() == 0 && isEmptyBadRecord)) {
+ row.update(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY, index);
+ } else {
+ try {
+ row.update(DataTypeUtil
+ .getBytesBasedOnDataTypeForNoDictionaryColumn(dimensionValue, dataType), index);
+ } catch (Throwable ex) {
+ if (dimensionValue.length() != 0 || isEmptyBadRecord) {
logHolder.setReason(
"The value " + " \"" + dimensionValue + "\"" + " with column name " + column
.getColName() + " and column data type " + dataType + " is not a valid "
+ dataType + " type.");
+ } else {
+ row.update(new byte[0], index);
}
}
}
- row.update(dimensionValue.getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)),
- index);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterImpl.java
index 856b6ac..0e14660 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterImpl.java
@@ -83,7 +83,8 @@ public class ParallelReadMergeSorterImpl extends AbstractMergeSorter {
sortParameters.getDimColCount(),
sortParameters.getComplexDimColCount(), sortParameters.getMeasureColCount(),
sortParameters.getNoDictionaryCount(), sortParameters.getAggType(),
- sortParameters.getNoDictionaryDimnesionColumn());
+ sortParameters.getNoDictionaryDimnesionColumn(),
+ sortParameters.getNoDictionarySortColumn());
}
@Override
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterWithBucketingImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterWithBucketingImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterWithBucketingImpl.java
index cb1b8fc..2db732c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterWithBucketingImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterWithBucketingImpl.java
@@ -147,7 +147,8 @@ public class ParallelReadMergeSorterWithBucketingImpl extends AbstractMergeSorte
new SingleThreadFinalSortFilesMerger(dataFolderLocation, sortParameters.getTableName(),
sortParameters.getDimColCount(), sortParameters.getComplexDimColCount(),
sortParameters.getMeasureColCount(), sortParameters.getNoDictionaryCount(),
- sortParameters.getAggType(), sortParameters.getNoDictionaryDimnesionColumn());
+ sortParameters.getAggType(), sortParameters.getNoDictionaryDimnesionColumn(),
+ this.sortParameters.getNoDictionarySortColumn());
return finalMerger;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeCarbonRowPage.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeCarbonRowPage.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeCarbonRowPage.java
index e682263..44f11f7 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeCarbonRowPage.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeCarbonRowPage.java
@@ -34,6 +34,8 @@ public class UnsafeCarbonRowPage {
private boolean[] noDictionaryDimensionMapping;
+ private boolean[] noDictionarySortColumnMapping;
+
private int dimensionSize;
private int measureSize;
@@ -52,9 +54,11 @@ public class UnsafeCarbonRowPage {
private boolean saveToDisk;
- public UnsafeCarbonRowPage(boolean[] noDictionaryDimensionMapping, int dimensionSize,
- int measureSize, char[] aggType, MemoryBlock memoryBlock, boolean saveToDisk) {
+ public UnsafeCarbonRowPage(boolean[] noDictionaryDimensionMapping,
+ boolean[] noDictionarySortColumnMapping, int dimensionSize, int measureSize, char[] aggType,
+ MemoryBlock memoryBlock, boolean saveToDisk) {
this.noDictionaryDimensionMapping = noDictionaryDimensionMapping;
+ this.noDictionarySortColumnMapping = noDictionarySortColumnMapping;
this.dimensionSize = dimensionSize;
this.measureSize = measureSize;
this.aggType = aggType;
@@ -324,4 +328,7 @@ public class UnsafeCarbonRowPage {
return noDictionaryDimensionMapping;
}
+ public boolean[] getNoDictionarySortColumnMapping() {
+ return noDictionarySortColumnMapping;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeSortDataRows.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeSortDataRows.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeSortDataRows.java
index df3825a..1219910 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeSortDataRows.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeSortDataRows.java
@@ -113,6 +113,7 @@ public class UnsafeSortDataRows {
public void initialize() throws CarbonSortKeyAndGroupByException {
MemoryBlock baseBlock = getMemoryBlock(inMemoryChunkSize);
this.rowPage = new UnsafeCarbonRowPage(parameters.getNoDictionaryDimnesionColumn(),
+ parameters.getNoDictionarySortColumn(),
parameters.getDimColCount() + parameters.getComplexDimColCount(),
parameters.getMeasureColCount(), parameters.getAggType(), baseBlock,
!UnsafeMemoryManager.INSTANCE.isMemoryAvailable());
@@ -178,6 +179,7 @@ public class UnsafeSortDataRows {
MemoryBlock memoryBlock = getMemoryBlock(inMemoryChunkSize);
boolean saveToDisk = !UnsafeMemoryManager.INSTANCE.isMemoryAvailable();
rowPage = new UnsafeCarbonRowPage(parameters.getNoDictionaryDimnesionColumn(),
+ parameters.getNoDictionarySortColumn(),
parameters.getDimColCount() + parameters.getComplexDimColCount(),
parameters.getMeasureColCount(), parameters.getAggType(), memoryBlock, saveToDisk);
bytesAdded += rowPage.addRow(rowBatch[i]);
@@ -237,12 +239,12 @@ public class UnsafeSortDataRows {
if (this.rowPage.getUsedSize() > 0) {
TimSort<UnsafeCarbonRow, IntPointerBuffer> timSort = new TimSort<>(
new UnsafeIntSortDataFormat(rowPage));
- if (parameters.getNoDictionaryCount() > 0) {
+ if (parameters.getNumberOfNoDictSortColumns() > 0) {
timSort.sort(rowPage.getBuffer(), 0, rowPage.getBuffer().getActualSize(),
new UnsafeRowComparator(rowPage));
} else {
timSort.sort(rowPage.getBuffer(), 0, rowPage.getBuffer().getActualSize(),
- new UnsafeRowComparatorForNormalDIms(parameters.getDimColCount(), rowPage));
+ new UnsafeRowComparatorForNormalDIms(rowPage));
}
unsafeInMemoryIntermediateFileMerger.addDataChunkToMerge(rowPage);
} else {
@@ -327,12 +329,13 @@ public class UnsafeSortDataRows {
long startTime = System.currentTimeMillis();
TimSort<UnsafeCarbonRow, IntPointerBuffer> timSort = new TimSort<>(
new UnsafeIntSortDataFormat(page));
- if (parameters.getNoDictionaryCount() > 0) {
+ // if sort_columns is not none, sort by sort_columns
+ if (parameters.getNumberOfNoDictSortColumns() > 0) {
timSort.sort(page.getBuffer(), 0, page.getBuffer().getActualSize(),
new UnsafeRowComparator(page));
} else {
timSort.sort(page.getBuffer(), 0, page.getBuffer().getActualSize(),
- new UnsafeRowComparatorForNormalDIms(parameters.getDimColCount(), page));
+ new UnsafeRowComparatorForNormalDIms(page));
}
if (rowPage.isSaveToDisk()) {
// create a new file every time
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/comparator/UnsafeRowComparator.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/comparator/UnsafeRowComparator.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/comparator/UnsafeRowComparator.java
index 8f048bd..1d60b11 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/comparator/UnsafeRowComparator.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/comparator/UnsafeRowComparator.java
@@ -27,14 +27,14 @@ import org.apache.carbondata.processing.newflow.sort.unsafe.holder.UnsafeCarbonR
public class UnsafeRowComparator implements Comparator<UnsafeCarbonRow> {
/**
- * noDictionaryColMaping mapping of dictionary dimensions and no dictionary dimensions.
+ * mapping of dictionary and no dictionary of sort_columns.
*/
- private boolean[] noDictionaryColMaping;
+ private boolean[] noDictionarySortColumnMaping;
private Object baseObject;
public UnsafeRowComparator(UnsafeCarbonRowPage rowPage) {
- this.noDictionaryColMaping = rowPage.getNoDictionaryDimensionMapping();
+ this.noDictionarySortColumnMaping = rowPage.getNoDictionarySortColumnMapping();
this.baseObject = rowPage.getDataBlock().getBaseObject();
}
@@ -47,7 +47,7 @@ public class UnsafeRowComparator implements Comparator<UnsafeCarbonRow> {
long rowB = rowR.address;
int sizeA = 0;
int sizeB = 0;
- for (boolean isNoDictionary : noDictionaryColMaping) {
+ for (boolean isNoDictionary : noDictionarySortColumnMaping) {
if (isNoDictionary) {
short aShort1 = CarbonUnsafe.unsafe.getShort(baseObject, rowA + sizeA);
byte[] byteArr1 = new byte[aShort1];
@@ -92,7 +92,7 @@ public class UnsafeRowComparator implements Comparator<UnsafeCarbonRow> {
long rowB = rowR.address;
int sizeA = 0;
int sizeB = 0;
- for (boolean isNoDictionary : noDictionaryColMaping) {
+ for (boolean isNoDictionary : noDictionarySortColumnMaping) {
if (isNoDictionary) {
short aShort1 = CarbonUnsafe.unsafe.getShort(baseObjectL, rowA + sizeA);
byte[] byteArr1 = new byte[aShort1];
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/comparator/UnsafeRowComparatorForNormalDIms.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/comparator/UnsafeRowComparatorForNormalDIms.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/comparator/UnsafeRowComparatorForNormalDIms.java
index 7448aee..4fd245f 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/comparator/UnsafeRowComparatorForNormalDIms.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/comparator/UnsafeRowComparatorForNormalDIms.java
@@ -27,11 +27,11 @@ public class UnsafeRowComparatorForNormalDIms implements Comparator<UnsafeCarbon
private Object baseObject;
- private int dimCount;
+ private int numberOfSortColumns;
- public UnsafeRowComparatorForNormalDIms(int dimCount, UnsafeCarbonRowPage rowPage) {
+ public UnsafeRowComparatorForNormalDIms(UnsafeCarbonRowPage rowPage) {
this.baseObject = rowPage.getDataBlock().getBaseObject();
- this.dimCount = dimCount;
+ this.numberOfSortColumns = rowPage.getNoDictionarySortColumnMapping().length;
}
/**
@@ -43,7 +43,7 @@ public class UnsafeRowComparatorForNormalDIms implements Comparator<UnsafeCarbon
long rowB = rowR.address;
int sizeA = 0;
int sizeB = 0;
- for (int i = 0; i < dimCount; i++) {
+ for (int i = 0; i < numberOfSortColumns; i++) {
int dimFieldA = CarbonUnsafe.unsafe.getInt(baseObject, rowA + sizeA);
sizeA += 4;
int dimFieldB = CarbonUnsafe.unsafe.getInt(baseObject, rowB + sizeB);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeFinalMergePageHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeFinalMergePageHolder.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeFinalMergePageHolder.java
index ed9e0a6..397de63 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeFinalMergePageHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeFinalMergePageHolder.java
@@ -45,13 +45,13 @@ public class UnsafeFinalMergePageHolder implements SortTempChunkHolder {
private int columnSize;
public UnsafeFinalMergePageHolder(UnsafeInMemoryIntermediateDataMerger merger,
- boolean[] noDictMapping, int columnSize) {
+ boolean[] noDictSortColumnMapping, int columnSize) {
this.actualSize = merger.getEntryCount();
this.mergedAddresses = merger.getMergedAddresses();
this.rowPageIndexes = merger.getRowPageIndexes();
this.rowPages = merger.getUnsafeCarbonRowPages();
LOGGER.audit("Processing unsafe inmemory rows page with size : " + actualSize);
- this.comparator = new NewRowComparator(noDictMapping);
+ this.comparator = new NewRowComparator(noDictSortColumnMapping);
this.columnSize = columnSize;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeInmemoryHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeInmemoryHolder.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeInmemoryHolder.java
index f491623..048f4f8 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeInmemoryHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeInmemoryHolder.java
@@ -41,11 +41,12 @@ public class UnsafeInmemoryHolder implements SortTempChunkHolder {
private int columnSize;
- public UnsafeInmemoryHolder(UnsafeCarbonRowPage rowPage, int columnSize) {
+ public UnsafeInmemoryHolder(UnsafeCarbonRowPage rowPage, int columnSize,
+ int numberOfSortColumns) {
this.actualSize = rowPage.getBuffer().getActualSize();
this.rowPage = rowPage;
LOGGER.audit("Processing unsafe inmemory rows page with size : " + actualSize);
- this.comparator = new NewRowComparator(rowPage.getNoDictionaryDimensionMapping());
+ this.comparator = new NewRowComparator(rowPage.getNoDictionarySortColumnMapping());
this.columnSize = columnSize;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
index de2b874..84aa029 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
@@ -153,7 +153,7 @@ public class UnsafeSortTempFileChunkHolder implements SortTempChunkHolder {
this.aggType = parameters.getAggType();
this.isNoDictionaryDimensionColumn = parameters.getNoDictionaryDimnesionColumn();
this.nullSetWordsLength = ((measureCount - 1) >> 6) + 1;
- comparator = new NewRowComparator(isNoDictionaryDimensionColumn);
+ comparator = new NewRowComparator(parameters.getNoDictionarySortColumn());
initialize();
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f945291/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeSingleThreadFinalSortFilesMerger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeSingleThreadFinalSortFilesMerger.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeSingleThreadFinalSortFilesMerger.java
index cd6b321..4055099 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeSingleThreadFinalSortFilesMerger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeSingleThreadFinalSortFilesMerger.java
@@ -132,7 +132,7 @@ public class UnsafeSingleThreadFinalSortFilesMerger extends CarbonIterator<Objec
SortTempChunkHolder sortTempFileChunkHolder = new UnsafeInmemoryHolder(rowPage,
parameters.getDimColCount() + parameters.getComplexDimColCount() + parameters
- .getMeasureColCount());
+ .getMeasureColCount(), parameters.getNumberOfSortColumns());
// initialize
sortTempFileChunkHolder.readRow();
@@ -143,7 +143,7 @@ public class UnsafeSingleThreadFinalSortFilesMerger extends CarbonIterator<Objec
for (final UnsafeInMemoryIntermediateDataMerger merger : merges) {
SortTempChunkHolder sortTempFileChunkHolder =
- new UnsafeFinalMergePageHolder(merger, parameters.getNoDictionaryDimnesionColumn(),
+ new UnsafeFinalMergePageHolder(merger, parameters.getNoDictionarySortColumn(),
parameters.getDimColCount() + parameters.getComplexDimColCount() + parameters
.getMeasureColCount());
[07/19] carbondata git commit: [CARBONDATA-909] Added option to
specify single pass load in data frame
Posted by ra...@apache.org.
[CARBONDATA-909] Added option to specify single pass load in data frame
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/26de62c1
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/26de62c1
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/26de62c1
Branch: refs/heads/master
Commit: 26de62c1ee4bc8ac9ceceb93fd028523bec44147
Parents: f2a5176
Author: Sanoj MG <sa...@gmail.com>
Authored: Wed Apr 12 22:32:51 2017 +0400
Committer: jackylk <ja...@huawei.com>
Committed: Sun May 14 20:40:12 2017 +0800
----------------------------------------------------------------------
.../testsuite/dataload/TestLoadDataFrame.scala | 32 ++++++++++++++++++++
.../spark/CarbonDataFrameWriter.scala | 3 +-
2 files changed, 34 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/26de62c1/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala
index f50620f..9179c08 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala
@@ -59,6 +59,8 @@ class TestLoadDataFrame extends QueryTest with BeforeAndAfterAll {
sql("DROP TABLE IF EXISTS carbon5")
sql("DROP TABLE IF EXISTS carbon6")
sql("DROP TABLE IF EXISTS carbon7")
+ sql("DROP TABLE IF EXISTS carbon8")
+ sql("DROP TABLE IF EXISTS carbon9")
}
@@ -167,6 +169,36 @@ class TestLoadDataFrame extends QueryTest with BeforeAndAfterAll {
)
}
+ test("test load dataframe with single pass enabled") {
+ // save dataframe to carbon file
+ df.write
+ .format("carbondata")
+ .option("tableName", "carbon8")
+ .option("tempCSV", "false")
+ .option("single_pass", "true")
+ .option("compress", "false")
+ .mode(SaveMode.Overwrite)
+ .save()
+ checkAnswer(
+ sql("select count(*) from carbon8 where c3 > 500"), Row(500)
+ )
+ }
+
+ test("test load dataframe with single pass disabled") {
+ // save dataframe to carbon file
+ df.write
+ .format("carbondata")
+ .option("tableName", "carbon9")
+ .option("tempCSV", "true")
+ .option("single_pass", "false")
+ .option("compress", "false")
+ .mode(SaveMode.Overwrite)
+ .save()
+ checkAnswer(
+ sql("select count(*) from carbon9 where c3 > 500"), Row(500)
+ )
+ }
+
override def afterAll {
dropTable
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/26de62c1/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala
index 36e2440..0d1b1df 100644
--- a/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala
+++ b/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala
@@ -193,7 +193,8 @@ class CarbonDataFrameWriter(val dataFrame: DataFrame) {
s"""
LOAD DATA INPATH '$csvFolder'
INTO TABLE ${options.dbName}.${options.tableName}
- OPTIONS ('FILEHEADER' = '${dataFrame.columns.mkString(",")}')
+ OPTIONS ('FILEHEADER' = '${dataFrame.columns.mkString(",")}',
+ 'SINGLE_PASS' = '${options.singlePass}')
"""
}
[14/19] carbondata git commit: [CARBONDATA-1015] Refactory write step
and add ColumnPage in data load This closes #852
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/datastore/page/ComplexColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/ComplexColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/ComplexColumnPage.java
new file mode 100644
index 0000000..024c341
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/ComplexColumnPage.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.datastore.page;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.carbondata.common.CarbonIterator;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+
+// Represent a complex column page, e.g. Array, Struct type column
+public class ComplexColumnPage extends ColumnPage {
+
+ // Holds data for all rows in this page in columnar layout.
+ // After the complex data expand, it is of type byte[][], the first level array in the byte[][]
+ // representing a sub-column in the complex type, which can be retrieved by giving the depth
+ // of the complex type.
+ // TODO: further optimize it to make it more memory efficient
+ private List<ArrayList<byte[]>> complexColumnData;
+
+ // depth is the number of column after complex type is expanded. It is from 1 to N
+ private final int depth;
+
+ public ComplexColumnPage(int pageSize, int depth) {
+ super(DataType.BYTE_ARRAY, pageSize);
+ this.depth = depth;
+ complexColumnData = new ArrayList<>(depth);
+ for (int i = 0; i < depth; i++) {
+ complexColumnData.add(new ArrayList<byte[]>(pageSize));
+ }
+ }
+
+ public void putComplexData(int rowId, int depth, List<byte[]> value) {
+ assert (depth <= this.depth);
+ ArrayList<byte[]> subColumnPage = complexColumnData.get(depth);
+ subColumnPage.addAll(value);
+ }
+
+ // iterate on the sub-column after complex type is expanded, return columnar page of
+ // each sub-column
+ public Iterator<byte[][]> iterator() {
+
+ return new CarbonIterator<byte[][]>() {
+ private int index = 0;
+ @Override public boolean hasNext() {
+ return index < depth;
+ }
+
+ @Override public byte[][] next() {
+ // convert the subColumnPage from ArrayList<byte[]> to byte[][]
+ ArrayList<byte[]> subColumnPage = complexColumnData.get(index);
+ index++;
+ return subColumnPage.toArray(new byte[subColumnPage.size()][]);
+ }
+ };
+ }
+
+ public int getDepth() {
+ return depth;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/datastore/page/FixLengthColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/FixLengthColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/FixLengthColumnPage.java
new file mode 100644
index 0000000..a56563e
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/FixLengthColumnPage.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.datastore.page;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.util.BitSet;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.util.DataTypeUtil;
+
+// Represent a columnar data in one page for one column.
+public class FixLengthColumnPage extends ColumnPage {
+
+ // Only one of following fields will be used
+ private byte[] byteData;
+ private short[] shortData;
+ private int[] intData;
+ private long[] longData;
+ private double[] doubleData;
+
+ private byte[][] byteArrayData;
+
+ // The index of the rowId whose value is null, will be set to 1
+ private BitSet nullBitSet;
+
+ public FixLengthColumnPage(DataType dataType, int pageSize) {
+ super(dataType, pageSize);
+ nullBitSet = new BitSet(pageSize);
+ switch (dataType) {
+ case SHORT:
+ case INT:
+ case LONG:
+ longData = new long[pageSize];
+ break;
+ case DOUBLE:
+ doubleData = new double[pageSize];
+ break;
+ case DECIMAL:
+ byteArrayData = new byte[pageSize][];
+ break;
+ default:
+ throw new RuntimeException("Unsupported data dataType: " + dataType);
+ }
+ }
+
+ public DataType getDataType() {
+ return dataType;
+ }
+
+ private void putByte(int rowId, byte value) {
+ byteData[rowId] = value;
+ }
+
+ private void putShort(int rowId, short value) {
+ shortData[rowId] = value;
+ }
+
+ private void putInt(int rowId, int value) {
+ intData[rowId] = value;
+ }
+
+ private void putLong(int rowId, long value) {
+ longData[rowId] = value;
+ }
+
+ private void putDouble(int rowId, double value) {
+ doubleData[rowId] = value;
+ }
+
+ // This method will do LV (length value) coded of input bytes
+ private void putDecimalBytes(int rowId, byte[] decimalInBytes) {
+ ByteBuffer byteBuffer = ByteBuffer.allocate(decimalInBytes.length +
+ CarbonCommonConstants.INT_SIZE_IN_BYTE);
+ byteBuffer.putInt(decimalInBytes.length);
+ byteBuffer.put(decimalInBytes);
+ byteBuffer.flip();
+ byteArrayData[rowId] = byteBuffer.array();
+ }
+
+ public void putData(int rowId, Object value) {
+ if (value == null) {
+ putNull(rowId);
+ return;
+ }
+ switch (dataType) {
+ case BYTE:
+ case SHORT:
+ case INT:
+ case LONG:
+ putLong(rowId, (long) value);
+ break;
+ case DOUBLE:
+ putDouble(rowId, (double) value);
+ break;
+ case DECIMAL:
+ putDecimalBytes(rowId, (byte[]) value);
+ break;
+ default:
+ throw new RuntimeException("unsupported data type: " + dataType);
+ }
+ updateStatistics(value);
+ }
+
+ private void putNull(int rowId) {
+ nullBitSet.set(rowId);
+ switch (dataType) {
+ case BYTE:
+ case SHORT:
+ case INT:
+ case LONG:
+ putLong(rowId, 0L);
+ break;
+ case DOUBLE:
+ putDouble(rowId, 0.0);
+ break;
+ case DECIMAL:
+ byte[] decimalInBytes = DataTypeUtil.bigDecimalToByte(BigDecimal.ZERO);
+ putDecimalBytes(rowId, decimalInBytes);
+ break;
+ }
+ }
+
+ public long[] getLongPage() {
+ return longData;
+ }
+
+ public double[] getDoublePage() {
+ return doubleData;
+ }
+
+ public byte[][] getDecimalPage() {
+ return byteArrayData;
+ }
+
+ public BitSet getNullBitSet() {
+ return nullBitSet;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPage.java
new file mode 100644
index 0000000..d5e9ce3
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPage.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.datastore.page;
+
+import org.apache.carbondata.core.metadata.datatype.DataType;
+
+// Represent a variable length columnar data in one page, e.g. for dictionary columns.
+public class VarLengthColumnPage extends ColumnPage {
+
+ // TODO: further optimizite it, to store length and data separately
+ private byte[][] byteArrayData;
+
+ public VarLengthColumnPage(int pageSize) {
+ super(DataType.BYTE_ARRAY, pageSize);
+ byteArrayData = new byte[pageSize][];
+ }
+
+ public void putByteArray(int rowId, byte[] value) {
+ byteArrayData[rowId] = value;
+ updateStatistics(value);
+ }
+
+ public byte[][] getByteArrayPage() {
+ return byteArrayData;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/datastore/page/compression/Compression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/compression/Compression.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/compression/Compression.java
new file mode 100644
index 0000000..c954a33
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/compression/Compression.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.datastore.page.compression;
+
+public interface Compression {
+ byte[] compress(byte[] input);
+ byte[] decompress(byte[] input);
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/ColumnCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/ColumnCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/ColumnCodec.java
new file mode 100644
index 0000000..e870ad6
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/ColumnCodec.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.datastore.page.encoding;
+
+import org.apache.carbondata.core.datastore.page.ColumnPage;
+
+/**
+ * Codec for a column page data, implementation should not keep state across pages,
+ * caller will use the same object to encode multiple pages.
+ */
+public interface ColumnCodec {
+
+ /** Codec name will be stored in BlockletHeader (DataChunk3) */
+ String getName();
+
+ byte[] encode(ColumnPage columnPage);
+
+ ColumnPage decode(byte[] encoded);
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DummyCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DummyCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DummyCodec.java
new file mode 100644
index 0000000..0dd23c7
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DummyCodec.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.datastore.page.encoding;
+
+import org.apache.carbondata.core.datastore.page.ColumnPage;
+
+public class DummyCodec implements ColumnCodec {
+ @Override
+ public String getName() {
+ return "DummyCodec";
+ }
+
+ @Override
+ public byte[] encode(ColumnPage columnPage) {
+ return null;
+ }
+
+ @Override
+ public ColumnPage decode(byte[] encoded) {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PageStatistics.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PageStatistics.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PageStatistics.java
new file mode 100644
index 0000000..3ecf1da
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PageStatistics.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.datastore.page.statistics;
+
+import java.math.BigDecimal;
+
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.util.DataTypeUtil;
+
+/** statics for one column page */
+public class PageStatistics {
+ private DataType dataType;
+
+ /** min and max value of the measures */
+ private Object min, max;
+
+ /**
+ * the unique value is the non-exist value in the row,
+ * and will be used as storage key for null values of measures
+ */
+ private Object uniqueValue;
+
+ /** decimal count of the measures */
+ private int decimal;
+
+ public PageStatistics(DataType dataType) {
+ this.dataType = dataType;
+ switch (dataType) {
+ case SHORT:
+ case INT:
+ case LONG:
+ max = Long.MIN_VALUE;
+ min = Long.MAX_VALUE;
+ uniqueValue = Long.MIN_VALUE;
+ break;
+ case DOUBLE:
+ max = Double.MIN_VALUE;
+ min = Double.MAX_VALUE;
+ uniqueValue = Double.MIN_VALUE;
+ break;
+ case DECIMAL:
+ max = new BigDecimal(Double.MIN_VALUE);
+ min = new BigDecimal(Double.MAX_VALUE);
+ uniqueValue = new BigDecimal(Double.MIN_VALUE);
+ break;
+ }
+ decimal = 0;
+ }
+
+ /**
+ * update the statistics for the input row
+ */
+ public void update(Object value) {
+ switch (dataType) {
+ case SHORT:
+ case INT:
+ case LONG:
+ max = ((long) max > (long) value) ? max : value;
+ min = ((long) min < (long) value) ? min : value;
+ uniqueValue = (long) min - 1;
+ break;
+ case DOUBLE:
+ max = ((double) max > (double) value) ? max : value;
+ min = ((double) min < (double) value) ? min : value;
+ int num = getDecimalCount((double) value);
+ decimal = decimal > num ? decimal : num;
+ uniqueValue = (double) min - 1;
+ break;
+ case DECIMAL:
+ BigDecimal decimalValue = DataTypeUtil.byteToBigDecimal((byte[]) value);
+ decimal = decimalValue.scale();
+ BigDecimal val = (BigDecimal) min;
+ uniqueValue = (val.subtract(new BigDecimal(1.0)));
+ break;
+ case ARRAY:
+ case STRUCT:
+ // for complex type column, writer is not going to use stats, so, do nothing
+ }
+ }
+
+ /**
+ * return no of digit after decimal
+ */
+ private int getDecimalCount(double value) {
+ String strValue = BigDecimal.valueOf(Math.abs(value)).toPlainString();
+ int integerPlaces = strValue.indexOf('.');
+ int decimalPlaces = 0;
+ if (-1 != integerPlaces) {
+ decimalPlaces = strValue.length() - integerPlaces - 1;
+ }
+ return decimalPlaces;
+ }
+
+ public Object getMin() {
+ return min;
+ }
+
+ public Object getMax() {
+ return max;
+ }
+
+ public Object getUniqueValue() {
+ return uniqueValue;
+ }
+
+ public int getDecimal() {
+ return decimal;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/StatisticsCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/StatisticsCollector.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/StatisticsCollector.java
new file mode 100644
index 0000000..f8b336c
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/StatisticsCollector.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.datastore.page.statistics;
+
+import org.apache.carbondata.core.datastore.page.ColumnPage;
+
+/**
+ * Calculate the statistics for a column page and blocklet
+ */
+public interface StatisticsCollector {
+
+ /**
+ * name will be stored in Header
+ */
+ String getName();
+
+ void startPage(int pageID);
+
+ void endPage(int pageID);
+
+ void startBlocklet(int blockletID);
+
+ void endBlocklet(int blockletID);
+
+ void startBlock(int blocklID);
+
+ void endBlock(int blockID);
+
+ /**
+ * Update the stats for the input batch
+ */
+ void update(ColumnPage batch);
+
+ /**
+ * Ouput will be written to DataChunk2 (page header)
+ */
+ byte[] getPageStatistisc();
+
+ /**
+ * Output will be written to DataChunk3 (blocklet header)
+ */
+ byte[] getBlockletStatistics();
+
+ /**
+ * Output will be written to Footer
+ */
+ byte[] getBlockStatistics();
+}
+
+
+
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/metadata/ValueEncoderMeta.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/ValueEncoderMeta.java b/core/src/main/java/org/apache/carbondata/core/metadata/ValueEncoderMeta.java
index 4a9007c..741b999 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/ValueEncoderMeta.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/ValueEncoderMeta.java
@@ -19,6 +19,9 @@ package org.apache.carbondata.core.metadata;
import java.io.Serializable;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+
/**
* DO NOT MODIFY THIS CLASS AND PACKAGE NAME, BECAUSE
* IT IS SERIALIZE TO STORE
@@ -78,7 +81,20 @@ public class ValueEncoderMeta implements Serializable {
this.decimal = decimal;
}
- public char getType() {
+ public DataType getType() {
+ switch (type) {
+ case CarbonCommonConstants.BIG_INT_MEASURE:
+ return DataType.LONG;
+ case CarbonCommonConstants.DOUBLE_MEASURE:
+ return DataType.DOUBLE;
+ case CarbonCommonConstants.BIG_DECIMAL_MEASURE:
+ return DataType.DECIMAL;
+ default:
+ throw new RuntimeException("Unexpected type: " + type);
+ }
+ }
+
+ public char getTypeInChar() {
return type;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataType.java
index d77406c..da13d5c 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataType.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataType.java
@@ -32,7 +32,11 @@ public enum DataType {
DECIMAL(8, "DECIMAL"),
ARRAY(9, "ARRAY"),
STRUCT(10, "STRUCT"),
- MAP(11, "MAP");
+ MAP(11, "MAP"),
+ BYTE(12, "BYTE"),
+
+ // internal use only
+ BYTE_ARRAY(13, "BYTE ARRAY");
private int precedenceOrder;
private String name ;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
index f4ab982..caba75f 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
@@ -39,6 +39,8 @@ public final class ByteUtil {
public static final String UTF8_CSN = StandardCharsets.UTF_8.name();
+ public static final byte[] ZERO_IN_BYTES = toBytes(0);
+
private ByteUtil() {
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
index 6398f30..6fe38e2 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
@@ -554,7 +554,7 @@ public class CarbonMetadataUtil {
Object[] minValue = new Object[encoderMetas.length];
int[] decimalLength = new int[encoderMetas.length];
Object[] uniqueValue = new Object[encoderMetas.length];
- char[] aggType = new char[encoderMetas.length];
+ DataType[] aggType = new DataType[encoderMetas.length];
byte[] dataTypeSelected = new byte[encoderMetas.length];
for (int i = 0; i < encoderMetas.length; i++) {
maxValue[i] = encoderMetas[i].getMaxValue();
@@ -827,25 +827,29 @@ public class CarbonMetadataUtil {
public static byte[] serializeEncodeMetaUsingByteBuffer(ValueEncoderMeta valueEncoderMeta) {
ByteBuffer buffer = null;
- if (valueEncoderMeta.getType() == CarbonCommonConstants.DOUBLE_MEASURE) {
- buffer = ByteBuffer.allocate(
- (CarbonCommonConstants.DOUBLE_SIZE_IN_BYTE * 3) + CarbonCommonConstants.INT_SIZE_IN_BYTE
- + 3);
- buffer.putChar(valueEncoderMeta.getType());
- buffer.putDouble((Double) valueEncoderMeta.getMaxValue());
- buffer.putDouble((Double) valueEncoderMeta.getMinValue());
- buffer.putDouble((Double) valueEncoderMeta.getUniqueValue());
- } else if (valueEncoderMeta.getType() == CarbonCommonConstants.BIG_INT_MEASURE) {
- buffer = ByteBuffer.allocate(
- (CarbonCommonConstants.LONG_SIZE_IN_BYTE * 3) + CarbonCommonConstants.INT_SIZE_IN_BYTE
- + 3);
- buffer.putChar(valueEncoderMeta.getType());
- buffer.putLong((Long) valueEncoderMeta.getMaxValue());
- buffer.putLong((Long) valueEncoderMeta.getMinValue());
- buffer.putLong((Long) valueEncoderMeta.getUniqueValue());
- } else {
- buffer = ByteBuffer.allocate(CarbonCommonConstants.INT_SIZE_IN_BYTE + 3);
- buffer.putChar(valueEncoderMeta.getType());
+ switch (valueEncoderMeta.getType()) {
+ case LONG:
+ buffer = ByteBuffer.allocate(
+ (CarbonCommonConstants.LONG_SIZE_IN_BYTE * 3) + CarbonCommonConstants.INT_SIZE_IN_BYTE
+ + 3);
+ buffer.putChar(valueEncoderMeta.getTypeInChar());
+ buffer.putLong((Long) valueEncoderMeta.getMaxValue());
+ buffer.putLong((Long) valueEncoderMeta.getMinValue());
+ buffer.putLong((Long) valueEncoderMeta.getUniqueValue());
+ break;
+ case DOUBLE:
+ buffer = ByteBuffer.allocate(
+ (CarbonCommonConstants.DOUBLE_SIZE_IN_BYTE * 3) + CarbonCommonConstants.INT_SIZE_IN_BYTE
+ + 3);
+ buffer.putChar(valueEncoderMeta.getTypeInChar());
+ buffer.putDouble((Double) valueEncoderMeta.getMaxValue());
+ buffer.putDouble((Double) valueEncoderMeta.getMinValue());
+ buffer.putDouble((Double) valueEncoderMeta.getUniqueValue());
+ break;
+ case DECIMAL:
+ buffer = ByteBuffer.allocate(CarbonCommonConstants.INT_SIZE_IN_BYTE + 3);
+ buffer.putChar(valueEncoderMeta.getTypeInChar());
+ break;
}
buffer.putInt(valueEncoderMeta.getDecimal());
buffer.put(valueEncoderMeta.getDataTypeSelected());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
index 92c85a1..496adff 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
@@ -749,5 +749,4 @@ public final class CarbonProperties {
}
return numberOfDeltaFilesThreshold;
}
-
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 99463de..8e4df1a 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -833,7 +833,7 @@ public final class CarbonUtil {
Object[] minValue = new Object[encodeMetaList.size()];
Object[] uniqueValue = new Object[encodeMetaList.size()];
int[] decimal = new int[encodeMetaList.size()];
- char[] type = new char[encodeMetaList.size()];
+ DataType[] type = new DataType[encodeMetaList.size()];
byte[] dataTypeSelected = new byte[encodeMetaList.size()];
/*
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/util/CompressionFinder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CompressionFinder.java b/core/src/main/java/org/apache/carbondata/core/util/CompressionFinder.java
index d931af6..732d053 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CompressionFinder.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CompressionFinder.java
@@ -16,8 +16,8 @@
*/
package org.apache.carbondata.core.util;
+import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.util.ValueCompressionUtil.COMPRESSION_TYPE;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
/**
@@ -37,7 +37,7 @@ public class CompressionFinder implements Comparable<CompressionFinder> {
private PRIORITY priority;
- private char measureStoreType;
+ private DataType measureStoreType;
/**
* CompressionFinder constructor.
@@ -47,7 +47,7 @@ public class CompressionFinder implements Comparable<CompressionFinder> {
* @param convertedDataType
*/
CompressionFinder(COMPRESSION_TYPE compType, DataType actualDataType,
- DataType convertedDataType, char measureStoreType) {
+ DataType convertedDataType, DataType measureStoreType) {
super();
this.compType = compType;
this.actualDataType = actualDataType;
@@ -65,7 +65,7 @@ public class CompressionFinder implements Comparable<CompressionFinder> {
*/
CompressionFinder(COMPRESSION_TYPE compType, DataType actualDataType, DataType convertedDataType,
- PRIORITY priority, char measureStoreType) {
+ PRIORITY priority, DataType measureStoreType) {
super();
this.actualDataType = actualDataType;
this.convertedDataType = convertedDataType;
@@ -155,7 +155,7 @@ public class CompressionFinder implements Comparable<CompressionFinder> {
return priority;
}
- public char getMeasureStoreType() {
+ public DataType getMeasureStoreType() {
return measureStoreType;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
index 80c9e72..e33d198 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -156,9 +156,6 @@ public final class DataTypeUtil {
}
}
- // bytes of 0 in BigDecimal
- public static final byte[] zeroBigDecimalBytes = bigDecimalToByte(BigDecimal.valueOf(0));
-
/**
* This method will convert a big decimal value to bytes
*
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/util/NodeHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/NodeHolder.java b/core/src/main/java/org/apache/carbondata/core/util/NodeHolder.java
index 69ed9f8..a37a9a7 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/NodeHolder.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/NodeHolder.java
@@ -57,16 +57,6 @@ public class NodeHolder {
private int[] keyLengths;
/**
- * dataAfterCompression
- */
- private short[][] dataAfterCompression;
-
- /**
- * indexMap
- */
- private short[][] indexMap;
-
- /**
* keyIndexBlockLenght
*/
private int[] keyBlockIndexLength;
@@ -86,11 +76,6 @@ public class NodeHolder {
private int[] dataIndexMapLength;
/**
- * dataIndexMap
- */
- private int[] dataIndexMapOffsets;
-
- /**
* compressedDataIndex
*/
private byte[][] compressedDataIndex;
@@ -120,19 +105,9 @@ public class NodeHolder {
private boolean[] aggBlocks;
/**
- * all columns max value
- */
- private byte[][] allMaxValue;
-
- /**
- * all column max value
- */
- private byte[][] allMinValue;
-
- /**
* true if given index is colgroup block
*/
- private boolean[] colGrpBlock;
+ private boolean[] colGrpBlocks;
/**
* bit set which will holds the measure
@@ -383,14 +358,14 @@ public class NodeHolder {
* @return
*/
public boolean[] getColGrpBlocks() {
- return this.colGrpBlock;
+ return this.colGrpBlocks;
}
/**
* @param colGrpBlock true if block is column group
*/
public void setColGrpBlocks(boolean[] colGrpBlock) {
- this.colGrpBlock = colGrpBlock;
+ this.colGrpBlocks = colGrpBlock;
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java b/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java
index c8a9397..5020acb 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java
@@ -28,10 +28,29 @@ import org.apache.carbondata.core.datastore.compression.MeasureMetaDataModel;
import org.apache.carbondata.core.datastore.compression.ReaderCompressModel;
import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
import org.apache.carbondata.core.datastore.compression.WriterCompressModel;
-import org.apache.carbondata.core.datastore.compression.decimal.*;
-import org.apache.carbondata.core.datastore.compression.nondecimal.*;
-import org.apache.carbondata.core.datastore.compression.none.*;
+import org.apache.carbondata.core.datastore.compression.decimal.CompressByteArray;
+import org.apache.carbondata.core.datastore.compression.decimal.CompressionMaxMinByte;
+import org.apache.carbondata.core.datastore.compression.decimal.CompressionMaxMinDefault;
+import org.apache.carbondata.core.datastore.compression.decimal.CompressionMaxMinInt;
+import org.apache.carbondata.core.datastore.compression.decimal.CompressionMaxMinLong;
+import org.apache.carbondata.core.datastore.compression.decimal.CompressionMaxMinShort;
+import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalByte;
+import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalDefault;
+import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalInt;
+import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalLong;
+import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalMaxMinByte;
+import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalMaxMinDefault;
+import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalMaxMinInt;
+import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalMaxMinLong;
+import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalMaxMinShort;
+import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalShort;
+import org.apache.carbondata.core.datastore.compression.none.CompressionNoneByte;
+import org.apache.carbondata.core.datastore.compression.none.CompressionNoneDefault;
+import org.apache.carbondata.core.datastore.compression.none.CompressionNoneInt;
+import org.apache.carbondata.core.datastore.compression.none.CompressionNoneLong;
+import org.apache.carbondata.core.datastore.compression.none.CompressionNoneShort;
import org.apache.carbondata.core.metadata.ValueEncoderMeta;
+import org.apache.carbondata.core.metadata.datatype.DataType;
public final class ValueCompressionUtil {
@@ -47,29 +66,28 @@ public final class ValueCompressionUtil {
* @see
*/
private static DataType getDataType(double value, int mantissa, byte dataTypeSelected) {
- DataType dataType = DataType.DATA_DOUBLE;
+ DataType dataType = DataType.DOUBLE;
if (mantissa == 0) {
if (value <= Byte.MAX_VALUE && value >= Byte.MIN_VALUE) {
- dataType = DataType.DATA_BYTE;
+ dataType = DataType.BYTE;
} else if (value <= Short.MAX_VALUE && value >= Short.MIN_VALUE) {
- dataType = DataType.DATA_SHORT;
+ dataType = DataType.SHORT;
} else if (value <= Integer.MAX_VALUE && value >= Integer.MIN_VALUE) {
- dataType = DataType.DATA_INT;
+ dataType = DataType.INT;
} else if (value <= Long.MAX_VALUE && value >= Long.MIN_VALUE) {
- dataType = DataType.DATA_LONG;
+ dataType = DataType.LONG;
}
} else {
if (dataTypeSelected == 1) {
if (value <= Float.MAX_VALUE && value >= Float.MIN_VALUE) {
float floatValue = (float) value;
if (floatValue - value != 0) {
- dataType = DataType.DATA_DOUBLE;
-
+ dataType = DataType.DOUBLE;
} else {
- dataType = DataType.DATA_FLOAT;
+ dataType = DataType.FLOAT;
}
} else if (value <= Double.MAX_VALUE && value >= Double.MIN_VALUE) {
- dataType = DataType.DATA_DOUBLE;
+ dataType = DataType.DOUBLE;
}
}
}
@@ -84,14 +102,14 @@ public final class ValueCompressionUtil {
* @see
*/
public static int getSize(DataType dataType) {
-
switch (dataType) {
- case DATA_BYTE:
+ case BOOLEAN:
+ case BYTE:
return 1;
- case DATA_SHORT:
+ case SHORT:
return 2;
- case DATA_INT:
- case DATA_FLOAT:
+ case INT:
+ case FLOAT:
return 4;
default:
return 8;
@@ -110,19 +128,17 @@ public final class ValueCompressionUtil {
* @see
*/
public static CompressionFinder getCompressionFinder(Object maxValue, Object minValue,
- int mantissa, char measureStoreType, byte dataTypeSelected) {
- // ''l' for long, 'n' for double
+ int mantissa, DataType measureStoreType, byte dataTypeSelected) {
switch (measureStoreType) {
- case 'b':
- return new CompressionFinder(COMPRESSION_TYPE.BIGDECIMAL, DataType.DATA_BYTE,
- DataType.DATA_BYTE, measureStoreType);
- case 'd':
+ case DECIMAL:
+ return new CompressionFinder(COMPRESSION_TYPE.BIGDECIMAL, DataType.BYTE,
+ DataType.BYTE, measureStoreType);
+ case SHORT:
+ case INT:
+ case LONG:
return getLongCompressorFinder(maxValue, minValue, mantissa, dataTypeSelected,
measureStoreType);
- case 'l':
- return new CompressionFinder(COMPRESSION_TYPE.ADAPTIVE,
- DataType.DATA_BIGINT, DataType.DATA_BIGINT, measureStoreType);
- case 'n':
+ case DOUBLE:
return getDoubleCompressorFinder(maxValue, minValue, mantissa, dataTypeSelected,
measureStoreType);
default:
@@ -131,7 +147,7 @@ public final class ValueCompressionUtil {
}
private static CompressionFinder getDoubleCompressorFinder(Object maxValue, Object minValue,
- int mantissa, byte dataTypeSelected, char measureStoreType) {
+ int mantissa, byte dataTypeSelected, DataType measureStoreType) {
//Here we should use the Max abs as max to getDatatype, let's say -1 and -10000000, -1 is max,
//but we can't use -1 to getDatatype, we should use -10000000.
double absMaxValue = Math.abs((double) maxValue) >= Math.abs((double) minValue) ?
@@ -145,13 +161,13 @@ public final class ValueCompressionUtil {
int adaptiveSize = getSize(adaptiveDataType);
int deltaSize = getSize(deltaDataType);
if (adaptiveSize > deltaSize) {
- return new CompressionFinder(COMPRESSION_TYPE.DELTA_DOUBLE, DataType.DATA_DOUBLE,
+ return new CompressionFinder(COMPRESSION_TYPE.DELTA_DOUBLE, DataType.DOUBLE,
deltaDataType, measureStoreType);
} else if (adaptiveSize < deltaSize) {
- return new CompressionFinder(COMPRESSION_TYPE.ADAPTIVE, DataType.DATA_DOUBLE,
+ return new CompressionFinder(COMPRESSION_TYPE.ADAPTIVE, DataType.DOUBLE,
deltaDataType, measureStoreType);
} else {
- return new CompressionFinder(COMPRESSION_TYPE.ADAPTIVE, DataType.DATA_DOUBLE,
+ return new CompressionFinder(COMPRESSION_TYPE.ADAPTIVE, DataType.DOUBLE,
adaptiveDataType, measureStoreType);
}
} else {
@@ -178,7 +194,7 @@ public final class ValueCompressionUtil {
}
private static CompressionFinder getLongCompressorFinder(Object maxValue, Object minValue,
- int mantissa, byte dataTypeSelected, char measureStoreType) {
+ int mantissa, byte dataTypeSelected, DataType measureStoreType) {
DataType adaptiveDataType = getDataType((long) maxValue, mantissa, dataTypeSelected);
int adaptiveSize = getSize(adaptiveDataType);
DataType deltaDataType = null;
@@ -186,20 +202,20 @@ public final class ValueCompressionUtil {
// consider the scenario when max and min value are equal to is long max and min value OR
// when the max and min value are resulting in a value greater than long max value, then
// it is not possible to determine the compression type.
- if (adaptiveDataType == DataType.DATA_LONG) {
- deltaDataType = DataType.DATA_BIGINT;
+ if (adaptiveDataType == DataType.LONG) {
+ deltaDataType = DataType.LONG;
} else {
deltaDataType = getDataType((long) maxValue - (long) minValue, mantissa, dataTypeSelected);
}
int deltaSize = getSize(deltaDataType);
if (adaptiveSize > deltaSize) {
- return new CompressionFinder(COMPRESSION_TYPE.DELTA_DOUBLE, DataType.DATA_BIGINT,
+ return new CompressionFinder(COMPRESSION_TYPE.DELTA_DOUBLE, DataType.LONG,
deltaDataType, measureStoreType);
} else if (adaptiveSize < deltaSize) {
- return new CompressionFinder(COMPRESSION_TYPE.ADAPTIVE, DataType.DATA_BIGINT,
+ return new CompressionFinder(COMPRESSION_TYPE.ADAPTIVE, DataType.LONG,
deltaDataType, measureStoreType);
} else {
- return new CompressionFinder(COMPRESSION_TYPE.ADAPTIVE, DataType.DATA_BIGINT,
+ return new CompressionFinder(COMPRESSION_TYPE.ADAPTIVE, DataType.LONG,
adaptiveDataType, measureStoreType);
}
}
@@ -234,7 +250,9 @@ public final class ValueCompressionUtil {
*/
public static ValueCompressor getValueCompressor(CompressionFinder compressorFinder) {
switch (compressorFinder.getMeasureStoreType()) {
- case 'd':
+ case SHORT:
+ case INT:
+ case LONG:
return new BigIntCompressor();
default:
return new DoubleCompressor();
@@ -295,7 +313,7 @@ public final class ValueCompressionUtil {
private static Object compressNone(DataType changedDataType, double[] value) {
int i = 0;
switch (changedDataType) {
- case DATA_BYTE:
+ case BYTE:
byte[] result = new byte[value.length];
for (double a : value) {
@@ -304,7 +322,7 @@ public final class ValueCompressionUtil {
}
return result;
- case DATA_SHORT:
+ case SHORT:
short[] shortResult = new short[value.length];
for (double a : value) {
@@ -313,7 +331,7 @@ public final class ValueCompressionUtil {
}
return shortResult;
- case DATA_INT:
+ case INT:
int[] intResult = new int[value.length];
for (double a : value) {
@@ -322,8 +340,7 @@ public final class ValueCompressionUtil {
}
return intResult;
- case DATA_LONG:
- case DATA_BIGINT:
+ case LONG:
long[] longResult = new long[value.length];
for (double a : value) {
@@ -332,7 +349,7 @@ public final class ValueCompressionUtil {
}
return longResult;
- case DATA_FLOAT:
+ case FLOAT:
float[] floatResult = new float[value.length];
for (double a : value) {
@@ -353,7 +370,7 @@ public final class ValueCompressionUtil {
private static Object compressMaxMin(DataType changedDataType, double[] value, double maxValue) {
int i = 0;
switch (changedDataType) {
- case DATA_BYTE:
+ case BYTE:
byte[] result = new byte[value.length];
for (double a : value) {
@@ -362,7 +379,7 @@ public final class ValueCompressionUtil {
}
return result;
- case DATA_SHORT:
+ case SHORT:
short[] shortResult = new short[value.length];
@@ -372,7 +389,7 @@ public final class ValueCompressionUtil {
}
return shortResult;
- case DATA_INT:
+ case INT:
int[] intResult = new int[value.length];
@@ -382,7 +399,7 @@ public final class ValueCompressionUtil {
}
return intResult;
- case DATA_LONG:
+ case LONG:
long[] longResult = new long[value.length];
@@ -392,7 +409,7 @@ public final class ValueCompressionUtil {
}
return longResult;
- case DATA_FLOAT:
+ case FLOAT:
float[] floatResult = new float[value.length];
@@ -422,7 +439,7 @@ public final class ValueCompressionUtil {
private static Object compressNonDecimal(DataType changedDataType, double[] value, int mantissa) {
int i = 0;
switch (changedDataType) {
- case DATA_BYTE:
+ case BYTE:
byte[] result = new byte[value.length];
for (double a : value) {
@@ -430,7 +447,7 @@ public final class ValueCompressionUtil {
i++;
}
return result;
- case DATA_SHORT:
+ case SHORT:
short[] shortResult = new short[value.length];
for (double a : value) {
@@ -438,7 +455,7 @@ public final class ValueCompressionUtil {
i++;
}
return shortResult;
- case DATA_INT:
+ case INT:
int[] intResult = new int[value.length];
@@ -448,7 +465,7 @@ public final class ValueCompressionUtil {
}
return intResult;
- case DATA_LONG:
+ case LONG:
long[] longResult = new long[value.length];
@@ -458,7 +475,7 @@ public final class ValueCompressionUtil {
}
return longResult;
- case DATA_FLOAT:
+ case FLOAT:
float[] floatResult = new float[value.length];
@@ -489,7 +506,7 @@ public final class ValueCompressionUtil {
int i = 0;
BigDecimal max = BigDecimal.valueOf(maxValue);
switch (changedDataType) {
- case DATA_BYTE:
+ case BYTE:
byte[] result = new byte[value.length];
@@ -501,7 +518,7 @@ public final class ValueCompressionUtil {
}
return result;
- case DATA_SHORT:
+ case SHORT:
short[] shortResult = new short[value.length];
@@ -513,7 +530,7 @@ public final class ValueCompressionUtil {
}
return shortResult;
- case DATA_INT:
+ case INT:
int[] intResult = new int[value.length];
@@ -525,7 +542,7 @@ public final class ValueCompressionUtil {
}
return intResult;
- case DATA_LONG:
+ case LONG:
long[] longResult = new long[value.length];
@@ -537,7 +554,7 @@ public final class ValueCompressionUtil {
}
return longResult;
- case DATA_FLOAT:
+ case FLOAT:
float[] floatResult = new float[value.length];
@@ -570,14 +587,13 @@ public final class ValueCompressionUtil {
public static ValueCompressionHolder getCompressionNone(DataType compDataType,
DataType actualDataType) {
switch (compDataType) {
- case DATA_BYTE:
+ case BYTE:
return new CompressionNoneByte(actualDataType);
- case DATA_SHORT:
+ case SHORT:
return new CompressionNoneShort(actualDataType);
- case DATA_INT:
+ case INT:
return new CompressionNoneInt(actualDataType);
- case DATA_LONG:
- case DATA_BIGINT:
+ case LONG:
return new CompressionNoneLong(actualDataType);
default:
return new CompressionNoneDefault(actualDataType);
@@ -590,13 +606,13 @@ public final class ValueCompressionUtil {
public static ValueCompressionHolder getCompressionDecimalMaxMin(
DataType compDataType, DataType actualDataType) {
switch (compDataType) {
- case DATA_BYTE:
+ case BYTE:
return new CompressionMaxMinByte(actualDataType);
- case DATA_SHORT:
+ case SHORT:
return new CompressionMaxMinShort(actualDataType);
- case DATA_INT:
+ case INT:
return new CompressionMaxMinInt(actualDataType);
- case DATA_LONG:
+ case LONG:
return new CompressionMaxMinLong(actualDataType);
default:
return new CompressionMaxMinDefault(actualDataType);
@@ -609,13 +625,13 @@ public final class ValueCompressionUtil {
public static ValueCompressionHolder getCompressionNonDecimal(
DataType compDataType) {
switch (compDataType) {
- case DATA_BYTE:
+ case BYTE:
return new CompressionNonDecimalByte();
- case DATA_SHORT:
+ case SHORT:
return new CompressionNonDecimalShort();
- case DATA_INT:
+ case INT:
return new CompressionNonDecimalInt();
- case DATA_LONG:
+ case LONG:
return new CompressionNonDecimalLong();
default:
return new CompressionNonDecimalDefault();
@@ -628,13 +644,13 @@ public final class ValueCompressionUtil {
public static ValueCompressionHolder getCompressionNonDecimalMaxMin(
DataType compDataType) {
switch (compDataType) {
- case DATA_BYTE:
+ case BYTE:
return new CompressionNonDecimalMaxMinByte();
- case DATA_SHORT:
+ case SHORT:
return new CompressionNonDecimalMaxMinShort();
- case DATA_INT:
+ case INT:
return new CompressionNonDecimalMaxMinInt();
- case DATA_LONG:
+ case LONG:
return new CompressionNonDecimalMaxMinLong();
default:
return new CompressionNonDecimalMaxMinDefault();
@@ -645,10 +661,10 @@ public final class ValueCompressionUtil {
* Create Value compression model for write path
*/
public static WriterCompressModel getWriterCompressModel(Object[] maxValue, Object[] minValue,
- int[] mantissa, Object[] uniqueValue, char[] aggType, byte[] dataTypeSelected) {
+ int[] mantissa, Object[] uniqueValue, DataType[] dataType, byte[] dataTypeSelected) {
MeasureMetaDataModel metaDataModel =
new MeasureMetaDataModel(minValue, maxValue, mantissa, maxValue.length, uniqueValue,
- aggType, dataTypeSelected);
+ dataType, dataTypeSelected);
return getWriterCompressModel(metaDataModel);
}
@@ -661,7 +677,7 @@ public final class ValueCompressionUtil {
Object[] maxValue = measureMDMdl.getMaxValue();
Object[] uniqueValue = measureMDMdl.getUniqueValue();
int[] mantissa = measureMDMdl.getMantissa();
- char[] type = measureMDMdl.getType();
+ DataType[] type = measureMDMdl.getType();
byte[] dataTypeSelected = measureMDMdl.getDataTypeSelected();
WriterCompressModel compressionModel = new WriterCompressModel();
DataType[] actualType = new DataType[measureCount];
@@ -772,20 +788,4 @@ public final class ValueCompressionUtil {
*/
BIGDECIMAL
}
-
- /**
- * use to identify the type of data.
- */
- public enum DataType {
- DATA_BYTE(),
- DATA_SHORT(),
- DATA_INT(),
- DATA_FLOAT(),
- DATA_LONG(),
- DATA_BIGINT(),
- DATA_DOUBLE(),
- DATA_BIGDECIMAL();
- DataType() {
- }
- }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java
index 2c6c890..ddcc8a4 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java
@@ -28,6 +28,8 @@ import org.apache.carbondata.core.metadata.ValueEncoderMeta;
import org.apache.carbondata.format.*;
import org.apache.carbondata.format.BlockletMinMaxIndex;
import org.apache.carbondata.format.ColumnSchema;
+import org.apache.carbondata.format.DataType;
+
import org.junit.BeforeClass;
import org.junit.Test;
@@ -169,7 +171,12 @@ public class CarbonMetadataUtilTest {
long[] longArr = { 1, 2, 3, 4, 5 };
byte[][] maxByteArr = { { 1, 2 }, { 3, 4 }, { 5, 6 }, { 2, 4 }, { 1, 2 } };
int[] cardinality = { 1, 2, 3, 4, 5 };
- char[] charArr = { 'a', 's', 'd', 'g', 'h' };
+ org.apache.carbondata.core.metadata.datatype.DataType[] dataType = {
+ org.apache.carbondata.core.metadata.datatype.DataType.INT,
+ org.apache.carbondata.core.metadata.datatype.DataType.INT,
+ org.apache.carbondata.core.metadata.datatype.DataType.INT,
+ org.apache.carbondata.core.metadata.datatype.DataType.INT,
+ org.apache.carbondata.core.metadata.datatype.DataType.INT };
org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema colSchema =
new org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema();
@@ -191,7 +198,7 @@ public class CarbonMetadataUtilTest {
writerCompressModel.setMinValue(objMinArr);
writerCompressModel.setDataTypeSelected(byteArr);
writerCompressModel.setMantissa(intArr);
- writerCompressModel.setType(charArr);
+ writerCompressModel.setType(dataType);
writerCompressModel.setUniqueValue(objMinArr);
BlockletInfoColumnar blockletInfoColumnar = new BlockletInfoColumnar();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java
index 6252ca1..3032085 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java
@@ -27,7 +27,7 @@ import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
import org.apache.carbondata.core.datastore.compression.decimal.*;
import org.apache.carbondata.core.datastore.compression.nondecimal.*;
import org.apache.carbondata.core.datastore.compression.none.*;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataType;
import org.junit.Test;
@@ -36,8 +36,8 @@ public class ValueCompressionUtilTest {
@Test public void testGetSize() {
DataType[] dataTypes =
- { DataType.DATA_BIGINT, DataType.DATA_INT, DataType.DATA_BYTE, DataType.DATA_SHORT,
- DataType.DATA_FLOAT };
+ { DataType.LONG, DataType.INT, DataType.BOOLEAN, DataType.SHORT,
+ DataType.FLOAT };
int[] expectedSizes = { 8, 4, 1, 2, 4 };
for (int i = 0; i < dataTypes.length; i++) {
assertEquals(expectedSizes[i], ValueCompressionUtil.getSize(dataTypes[i]));
@@ -48,7 +48,7 @@ public class ValueCompressionUtilTest {
double[] values = { 25, 12, 22 };
int[] result = (int[]) ValueCompressionUtil
.getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_DOUBLE, values,
- DataType.DATA_INT, 22, 0);
+ DataType.INT, 22, 0);
int[] expectedResult = { -3, 10, 0 };
for (int i = 0; i < result.length; i++) {
assertEquals(result[i], expectedResult[i]);
@@ -59,7 +59,7 @@ public class ValueCompressionUtilTest {
double[] values = { 20, 21, 22 };
byte[] result = (byte[]) ValueCompressionUtil
.getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_DOUBLE, values,
- DataType.DATA_BYTE, 22, 0);
+ DataType.BYTE, 22, 0);
byte[] expectedResult = { 2, 1, 0 };
for (int i = 0; i < result.length; i++) {
assertEquals(result[i], expectedResult[i]);
@@ -70,7 +70,7 @@ public class ValueCompressionUtilTest {
double[] values = { 200, 21, 22 };
short[] result = (short[]) ValueCompressionUtil
.getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_DOUBLE, values,
- DataType.DATA_SHORT, 22, 0);
+ DataType.SHORT, 22, 0);
short[] expectedResult = { -178, 1, 0 };
for (int i = 0; i < result.length; i++) {
assertEquals(result[i], expectedResult[i]);
@@ -81,7 +81,7 @@ public class ValueCompressionUtilTest {
double[] values = { 2000, 2100, 2002 };
long[] result = (long[]) ValueCompressionUtil
.getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_DOUBLE, values,
- DataType.DATA_LONG, 2125, 0);
+ DataType.LONG, 2125, 0);
long[] expectedResult = { 125, 25, 123 };
for (int i = 0; i < result.length; i++) {
assertEquals(result[i], expectedResult[i]);
@@ -92,7 +92,7 @@ public class ValueCompressionUtilTest {
double[] values = { 20.121, 21.223, 22.345 };
float[] result = (float[]) ValueCompressionUtil
.getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_DOUBLE, values,
- DataType.DATA_FLOAT, 22.345, 3);
+ DataType.FLOAT, 22.345, 3);
float[] expectedResult = { 2.224f, 1.122f, 0f };
for (int i = 0; i < result.length; i++) {
assertTrue(result[i]-expectedResult[i]==0);
@@ -103,7 +103,7 @@ public class ValueCompressionUtilTest {
double[] values = { 20.121, 21.223, 22.345 };
double[] result = (double[]) ValueCompressionUtil
.getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_DOUBLE, values,
- DataType.DATA_DOUBLE, 102.345, 3);
+ DataType.DOUBLE, 102.345, 3);
double[] expectedResult = { 82.224, 81.122, 80.0 };
for (int i = 0; i < result.length; i++) {
assertTrue(result[i]-expectedResult[i]==0);
@@ -114,7 +114,7 @@ public class ValueCompressionUtilTest {
double[] values = { 20.121, 21.223, 22.345 };
long[] result = (long[]) ValueCompressionUtil
.getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE, values,
- DataType.DATA_BIGINT, 22, 0);
+ DataType.LONG, 22, 0);
long[] expectedResult = { 20, 21, 22 };
for (int i = 0; i < result.length; i++) {
assertEquals(result[i], expectedResult[i]);
@@ -124,7 +124,7 @@ public class ValueCompressionUtilTest {
@Test public void testToGetCompressedValuesWithCompressionTypeNoneForDataByte() {
double[] values = { 20, 21, 22 };
byte[] result = (byte[]) ValueCompressionUtil
- .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE, values, DataType.DATA_BYTE,
+ .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE, values, DataType.BYTE,
22, 0);
byte[] expectedResult = { 20, 21, 22 };
for (int i = 0; i < result.length; i++) {
@@ -136,7 +136,7 @@ public class ValueCompressionUtilTest {
double[] values = { 200000, 21, 22 };
short[] result = (short[]) ValueCompressionUtil
.getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE, values,
- DataType.DATA_SHORT, 22, 0);
+ DataType.SHORT, 22, 0);
short[] expectedResult = { 3392, 21, 22 };
for (int i = 0; i < result.length; i++) {
assertEquals(result[i], expectedResult[i]);
@@ -146,7 +146,7 @@ public class ValueCompressionUtilTest {
@Test public void testToGetCompressedValuesWithCompressionTypeNoneForDataInt() {
double[] values = { 20, 21, 22 };
int[] result = (int[]) ValueCompressionUtil
- .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE, values, DataType.DATA_INT,
+ .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE, values, DataType.INT,
22, 0);
int[] expectedResult = { 20, 21, 22 };
for (int i = 0; i < result.length; i++) {
@@ -157,7 +157,7 @@ public class ValueCompressionUtilTest {
@Test public void testToGetCompressedValuesWithCompressionTypeNoneForDataLong() {
double[] values = { 20, 21, 22 };
long[] result = (long[]) ValueCompressionUtil
- .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE, values, DataType.DATA_LONG,
+ .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE, values, DataType.LONG,
22, 0);
long[] expectedResult = { 20, 21, 22 };
for (int i = 0; i < result.length; i++) {
@@ -169,7 +169,7 @@ public class ValueCompressionUtilTest {
double[] values = { 20.121, 21.223, 22.345 };
float[] result = (float[]) ValueCompressionUtil
.getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE, values,
- DataType.DATA_FLOAT, 22, 3);
+ DataType.FLOAT, 22, 3);
float[] expectedResult = { 20.121f, 21.223f, 22.345f };
for (int i = 0; i < result.length; i++) {
assertEquals(result[i], expectedResult[i],3);
@@ -180,7 +180,7 @@ public class ValueCompressionUtilTest {
double[] values = { 20.121, 21.223, 22.345 };
double[] result = (double[]) ValueCompressionUtil
.getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE, values,
- DataType.DATA_DOUBLE, 22, 3);
+ DataType.DOUBLE, 22, 3);
double[] expectedResult = { 20.121, 21.223, 22.345 };
for (int i = 0; i < result.length; i++) {
assertEquals(result[i], expectedResult[i],3);
@@ -191,7 +191,7 @@ public class ValueCompressionUtilTest {
double[] values = { 20.1, 21.2, 22.3 };
float[] result = (float[]) ValueCompressionUtil
.getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.BIGINT, values,
- DataType.DATA_FLOAT, 22, 1);
+ DataType.FLOAT, 22, 1);
float[] expectedResult = { 201f, 212f, 223f };
for (int i = 0; i < result.length; i++) {
assertEquals(result[i], expectedResult[i],0);
@@ -202,7 +202,7 @@ public class ValueCompressionUtilTest {
double[] values = { 20.1, 21.2, 22.3 };
byte[] result = (byte[]) ValueCompressionUtil
.getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.BIGINT, values,
- DataType.DATA_BYTE, 22, 1);
+ DataType.BYTE, 22, 1);
byte[] expectedResult = { -55, -44, -33 };
for (int i = 0; i < result.length; i++) {
assertEquals(result[i], expectedResult[i]);
@@ -213,7 +213,7 @@ public class ValueCompressionUtilTest {
double[] values = { 20.1, 21.2, 22.3 };
short[] result = (short[]) ValueCompressionUtil
.getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.BIGINT, values,
- DataType.DATA_SHORT, 22, 1);
+ DataType.SHORT, 22, 1);
short[] expectedResult = { 201, 212, 223 };
for (int i = 0; i < result.length; i++) {
assertEquals(result[i], expectedResult[i]);
@@ -224,7 +224,7 @@ public class ValueCompressionUtilTest {
double[] values = { 20.1, 21.2, 22.3 };
int[] result = (int[]) ValueCompressionUtil
.getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.BIGINT, values,
- DataType.DATA_INT, 22, 1);
+ DataType.INT, 22, 1);
int[] expectedResult = { 201, 212, 223 };
for (int i = 0; i < result.length; i++) {
assertEquals(result[i], expectedResult[i]);
@@ -235,7 +235,7 @@ public class ValueCompressionUtilTest {
double[] values = { 20.1, 21.2, 22.3 };
long[] result = (long[]) ValueCompressionUtil
.getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.BIGINT, values,
- DataType.DATA_LONG, 22, 1);
+ DataType.LONG, 22, 1);
long[] expectedResult = { 201, 212, 223 };
for (int i = 0; i < result.length; i++) {
assertEquals(result[i], expectedResult[i]);
@@ -246,7 +246,7 @@ public class ValueCompressionUtilTest {
double[] values = { 20.1, 21.2, 22.3 };
double[] result = (double[]) ValueCompressionUtil
.getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.BIGINT, values,
- DataType.DATA_DOUBLE, 22, 1);
+ DataType.DOUBLE, 22, 1);
double[] expectedResult = { 201, 212, 223 };
for (int i = 0; i < result.length; i++) {
assertEquals(result[i], expectedResult[i],0);
@@ -257,7 +257,7 @@ public class ValueCompressionUtilTest {
double[] values = { 20, 21, 22 };
byte[] result = (byte[]) ValueCompressionUtil
.getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_NON_DECIMAL, values,
- DataType.DATA_BYTE, 22, 1);
+ DataType.BYTE, 22, 1);
byte[] expectedResult = { 20, 10, 0 };
for (int i = 0; i < result.length; i++) {
assertEquals(result[i], expectedResult[i]);
@@ -268,7 +268,7 @@ public class ValueCompressionUtilTest {
double[] values = { 20, 21, 22 };
int[] result = (int[]) ValueCompressionUtil
.getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_NON_DECIMAL, values,
- DataType.DATA_INT, 22, 1);
+ DataType.INT, 22, 1);
int[] expectedResult = { 20, 10, 0 };
for (int i = 0; i < result.length; i++) {
assertEquals(result[i], expectedResult[i]);
@@ -279,7 +279,7 @@ public class ValueCompressionUtilTest {
double[] values = { 20, 21, 22 };
double[] result = (double[]) ValueCompressionUtil
.getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_NON_DECIMAL, values,
- DataType.DATA_DOUBLE, 22, 1);
+ DataType.DOUBLE, 22, 1);
double[] expectedResult = { 20, 10, 0 };
for (int i = 0; i < result.length; i++) {
assertEquals(result[i], expectedResult[i],0);
@@ -290,7 +290,7 @@ public class ValueCompressionUtilTest {
double[] values = { 20000, 21, 22 };
short[] result = (short[]) ValueCompressionUtil
.getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_NON_DECIMAL, values,
- DataType.DATA_SHORT, 22, 1);
+ DataType.SHORT, 22, 1);
short[] expectedResult = { -3172, 10, 0 };
for (int i = 0; i < result.length; i++) {
assertEquals(result[i], expectedResult[i]);
@@ -301,7 +301,7 @@ public class ValueCompressionUtilTest {
double[] values = { 20, 21, 22 };
long[] result = (long[]) ValueCompressionUtil
.getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_NON_DECIMAL, values,
- DataType.DATA_LONG, 22, 1);
+ DataType.LONG, 22, 1);
long[] expectedResult = { 20, 10, 0 };
for (int i = 0; i < result.length; i++) {
assertEquals(result[i], expectedResult[i]);
@@ -312,7 +312,7 @@ public class ValueCompressionUtilTest {
double[] values = { 20, 21, 22 };
float[] result = (float[]) ValueCompressionUtil
.getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_NON_DECIMAL, values,
- DataType.DATA_FLOAT, 22, 1);
+ DataType.FLOAT, 22, 1);
float[] expectedResult = { 20f, 10f, 0f };
for (int i = 0; i < result.length; i++) {
assertEquals(result[i], expectedResult[i],0);
@@ -321,127 +321,127 @@ public class ValueCompressionUtilTest {
@Test public void testToUnCompressNone() {
ValueCompressionHolder result =
- ValueCompressionUtil.getCompressionNone(DataType.DATA_BIGINT, DataType.DATA_BIGINT);
+ ValueCompressionUtil.getCompressionNone(DataType.LONG, DataType.LONG);
assertEquals(result.getClass(), CompressionNoneLong.class);
}
@Test public void testToUnCompressNoneForByte() {
ValueCompressionHolder result =
- ValueCompressionUtil.getCompressionNone(DataType.DATA_BYTE, DataType.DATA_FLOAT);
+ ValueCompressionUtil.getCompressionNone(DataType.BYTE, DataType.FLOAT);
assertEquals(result.getClass(), CompressionNoneByte.class);
}
@Test public void testToUnCompressNoneForLong() {
ValueCompressionHolder result =
- ValueCompressionUtil.getCompressionNone(DataType.DATA_LONG, DataType.DATA_FLOAT);
+ ValueCompressionUtil.getCompressionNone(DataType.LONG, DataType.FLOAT);
assertEquals(result.getClass(), CompressionNoneLong.class);
}
@Test public void testToUnCompressNoneForShort() {
ValueCompressionHolder result =
- ValueCompressionUtil.getCompressionNone(DataType.DATA_SHORT, DataType.DATA_FLOAT);
+ ValueCompressionUtil.getCompressionNone(DataType.SHORT, DataType.FLOAT);
assertEquals(result.getClass(), CompressionNoneShort.class);
}
@Test public void testToUnCompressNoneForInt() {
ValueCompressionHolder result =
- ValueCompressionUtil.getCompressionNone(DataType.DATA_INT, DataType.DATA_FLOAT);
+ ValueCompressionUtil.getCompressionNone(DataType.INT, DataType.FLOAT);
assertEquals(result.getClass(), CompressionNoneInt.class);
}
@Test public void testToUnCompressNoneForDouble() {
ValueCompressionHolder result =
- ValueCompressionUtil.getCompressionNone(DataType.DATA_DOUBLE, DataType.DATA_FLOAT);
+ ValueCompressionUtil.getCompressionNone(DataType.DOUBLE, DataType.FLOAT);
assertEquals(result.getClass(), CompressionNoneDefault.class);
}
@Test public void testToUnCompressMaxMinForDouble() {
ValueCompressionHolder result =
- ValueCompressionUtil.getCompressionDecimalMaxMin(DataType.DATA_DOUBLE, null);
+ ValueCompressionUtil.getCompressionDecimalMaxMin(DataType.DOUBLE, null);
assertEquals(result.getClass(), CompressionMaxMinDefault.class);
}
@Test public void testToUnCompressMaxMinForInt() {
ValueCompressionHolder result =
- ValueCompressionUtil.getCompressionDecimalMaxMin(DataType.DATA_INT, null);
+ ValueCompressionUtil.getCompressionDecimalMaxMin(DataType.INT, null);
assertEquals(result.getClass(), CompressionMaxMinInt.class);
}
@Test public void testToUnCompressMaxMinForLong() {
ValueCompressionHolder result =
- ValueCompressionUtil.getCompressionDecimalMaxMin(DataType.DATA_LONG, null);
+ ValueCompressionUtil.getCompressionDecimalMaxMin(DataType.LONG, null);
assertEquals(result.getClass(), CompressionMaxMinLong.class);
}
@Test public void testToUnCompressMaxMinForByte() {
ValueCompressionHolder result =
- ValueCompressionUtil.getCompressionDecimalMaxMin(DataType.DATA_BYTE, null);
+ ValueCompressionUtil.getCompressionDecimalMaxMin(DataType.BYTE, null);
assertEquals(result.getClass(), CompressionMaxMinByte.class);
}
@Test public void testToUnCompressMaxMinForShort() {
ValueCompressionHolder result =
- ValueCompressionUtil.getCompressionDecimalMaxMin(DataType.DATA_SHORT, null);
+ ValueCompressionUtil.getCompressionDecimalMaxMin(DataType.SHORT, null);
assertEquals(result.getClass(), CompressionMaxMinShort.class);
}
@Test public void testToUnCompressNonDecimalForDouble() {
ValueCompressionHolder result =
- ValueCompressionUtil.getCompressionNonDecimal(DataType.DATA_DOUBLE);
+ ValueCompressionUtil.getCompressionNonDecimal(DataType.DOUBLE);
assertEquals(result.getClass(), CompressionNonDecimalDefault.class);
}
@Test public void testToUnCompressNonDecimalForInt() {
ValueCompressionHolder result =
- ValueCompressionUtil.getCompressionNonDecimal(DataType.DATA_INT);
+ ValueCompressionUtil.getCompressionNonDecimal(DataType.INT);
assertEquals(result.getClass(), CompressionNonDecimalInt.class);
}
@Test public void testToUnCompressNonDecimalForLong() {
ValueCompressionHolder result =
- ValueCompressionUtil.getCompressionNonDecimal(DataType.DATA_LONG);
+ ValueCompressionUtil.getCompressionNonDecimal(DataType.LONG);
assertEquals(result.getClass(), CompressionNonDecimalLong.class);
}
@Test public void testToUnCompressNonDecimalForByte() {
ValueCompressionHolder result =
- ValueCompressionUtil.getCompressionNonDecimal(DataType.DATA_BYTE);
+ ValueCompressionUtil.getCompressionNonDecimal(DataType.BYTE);
assertEquals(result.getClass(), CompressionNonDecimalByte.class);
}
@Test public void testToUnCompressNonDecimalForShort() {
ValueCompressionHolder result =
- ValueCompressionUtil.getCompressionNonDecimal(DataType.DATA_SHORT);
+ ValueCompressionUtil.getCompressionNonDecimal(DataType.SHORT);
assertEquals(result.getClass(), CompressionNonDecimalShort.class);
}
@Test public void testToUnCompressNonDecimalMaxMinForDouble() {
ValueCompressionHolder result =
- ValueCompressionUtil.getCompressionNonDecimalMaxMin(DataType.DATA_DOUBLE);
+ ValueCompressionUtil.getCompressionNonDecimalMaxMin(DataType.DOUBLE);
assertEquals(result.getClass(), CompressionNonDecimalMaxMinDefault.class);
}
@Test public void testToUnCompressNonDecimalMaxMinForInt() {
ValueCompressionHolder result =
- ValueCompressionUtil.getCompressionNonDecimalMaxMin(DataType.DATA_INT);
+ ValueCompressionUtil.getCompressionNonDecimalMaxMin(DataType.INT);
assertEquals(result.getClass(), CompressionNonDecimalMaxMinInt.class);
}
@Test public void testToUnCompressNonDecimalMaxMinForLong() {
ValueCompressionHolder result =
- ValueCompressionUtil.getCompressionNonDecimalMaxMin(DataType.DATA_LONG);
+ ValueCompressionUtil.getCompressionNonDecimalMaxMin(DataType.LONG);
assertEquals(result.getClass(), CompressionNonDecimalMaxMinLong.class);
}
@Test public void testToUnCompressNonDecimalMaxMinForByte() {
ValueCompressionHolder result =
- ValueCompressionUtil.getCompressionNonDecimalMaxMin(DataType.DATA_BYTE);
+ ValueCompressionUtil.getCompressionNonDecimalMaxMin(DataType.BYTE);
assertEquals(result.getClass(), CompressionNonDecimalMaxMinByte.class);
}
@Test public void testToUnCompressNonDecimalMaxMinForShort() {
ValueCompressionHolder result =
- ValueCompressionUtil.getCompressionNonDecimalMaxMin(DataType.DATA_SHORT);
+ ValueCompressionUtil.getCompressionNonDecimalMaxMin(DataType.SHORT);
assertEquals(result.getClass(), CompressionNonDecimalMaxMinShort.class);
}
@@ -490,7 +490,7 @@ public class ValueCompressionUtilTest {
Object[] minValues = { 1L, 2L, 3L };
int[] decimalLength = { 0, 0, 0 };
Object[] uniqueValues = { 5, new Long[]{2L,4L}, 2L};
- char[] types = { 'l', 'l', 'l' };
+ DataType[] types = { DataType.LONG, DataType.LONG, DataType.LONG };
byte[] dataTypeSelected = { 1, 2, 4 };
MeasureMetaDataModel measureMetaDataModel =
new MeasureMetaDataModel(maxValues, minValues, decimalLength, 3, uniqueValues, types,
@@ -510,7 +510,7 @@ public class ValueCompressionUtilTest {
Object[] minValues = { 1.0 };
int[] decimalLength = { 0 };
Object[] uniqueValues = { 5 };
- char[] types = { 'n' };
+ DataType[] types = { DataType.DOUBLE };
byte[] dataTypeSelected = { 1 };
MeasureMetaDataModel measureMetaDataModel =
new MeasureMetaDataModel(maxValues, minValues, decimalLength, 1, uniqueValues, types,
@@ -526,7 +526,7 @@ public class ValueCompressionUtilTest {
Object[] minValues = { 32500.00 };
int[] decimalLength = { 0 };
Object[] uniqueValues = { 5 };
- char[] types = { 'n' };
+ DataType[] types = { DataType.DOUBLE };
byte[] dataTypeSelected = { 1 };
MeasureMetaDataModel measureMetaDataModel =
new MeasureMetaDataModel(maxValues, minValues, decimalLength, 1, uniqueValues, types,
@@ -542,7 +542,7 @@ public class ValueCompressionUtilTest {
Object[] minValues = { 1111078433.0 };
int[] decimalLength = { 0 };
Object[] uniqueValues = { 5 };
- char[] types = { 'n' };
+ DataType[] types = { DataType.DOUBLE };
byte[] dataTypeSelected = { 1 };
MeasureMetaDataModel measureMetaDataModel =
new MeasureMetaDataModel(maxValues, minValues, decimalLength, 1, uniqueValues, types,
@@ -558,7 +558,7 @@ public class ValueCompressionUtilTest {
Object[] minValues = { 32744.0 };
int[] decimalLength = { 0 };
Object[] uniqueValues = { 5 };
- char[] types = { 'n' };
+ DataType[] types = { DataType.DOUBLE};
byte[] dataTypeSelected = { 1 };
MeasureMetaDataModel measureMetaDataModel =
new MeasureMetaDataModel(maxValues, minValues, decimalLength, 1, uniqueValues, types,
@@ -574,7 +574,7 @@ public class ValueCompressionUtilTest {
Object[] minValues = { 32744.0 };
int[] decimalLength = { 1 };
Object[] uniqueValues = { 5 };
- char[] types = { 'n' };
+ DataType[] types = { DataType.DOUBLE };
byte[] dataTypeSelected = { 1 };
MeasureMetaDataModel measureMetaDataModel =
new MeasureMetaDataModel(maxValues, minValues, decimalLength, 1, uniqueValues, types,
@@ -590,7 +590,7 @@ public class ValueCompressionUtilTest {
Object[] minValues = { 32744.0 };
int[] decimalLength = { 1 };
Object[] uniqueValues = { 5 };
- char[] types = { 'n' };
+ DataType[] types = { DataType.DOUBLE };
byte[] dataTypeSelected = { 0 };
MeasureMetaDataModel measureMetaDataModel =
new MeasureMetaDataModel(maxValues, minValues, decimalLength, 1, uniqueValues, types,
@@ -606,7 +606,7 @@ public class ValueCompressionUtilTest {
Object[] minValues = { 32744.0 };
int[] decimalLength = { 1 };
Object[] uniqueValues = { 5 };
- char[] types = { 'n' };
+ DataType[] types = { DataType.DOUBLE };
byte[] dataTypeSelected = { 1 };
MeasureMetaDataModel measureMetaDataModel =
new MeasureMetaDataModel(maxValues, minValues, decimalLength, 1, uniqueValues, types,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/test/java/org/apache/carbondata/core/writer/CarbonFooterWriterTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/writer/CarbonFooterWriterTest.java b/core/src/test/java/org/apache/carbondata/core/writer/CarbonFooterWriterTest.java
index d02e25f..480ed04 100644
--- a/core/src/test/java/org/apache/carbondata/core/writer/CarbonFooterWriterTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/writer/CarbonFooterWriterTest.java
@@ -184,7 +184,7 @@ public class CarbonFooterWriterTest extends TestCase{
compressionModel.setMaxValue(new Object[] { 44d, 55d });
compressionModel.setMinValue(new Object[] { 0d, 0d });
compressionModel.setMantissa(new int[] { 0, 0 });
- compressionModel.setType(new char[] { 'n', 'n' });
+ compressionModel.setType(new DataType[] { DataType.DOUBLE, DataType.DOUBLE });
compressionModel.setUniqueValue(new Object[] { 0d, 0d });
compressionModel.setDataTypeSelected(new byte[2]);
infoColumnar.setCompressionModel(compressionModel);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
index a55c39e..8f8e891 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
@@ -44,6 +44,7 @@ object CarbonSessionExample {
.master("local")
.appName("CarbonSessionExample")
.config("spark.sql.warehouse.dir", warehouse)
+ .config("spark.driver.host", "localhost")
.getOrCreateCarbonSession(storeLocation, metastoredb)
spark.sparkContext.setLogLevel("WARN")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/format/src/main/thrift/carbondata.thrift
----------------------------------------------------------------------
diff --git a/format/src/main/thrift/carbondata.thrift b/format/src/main/thrift/carbondata.thrift
index 937108c..b4cbc4e 100644
--- a/format/src/main/thrift/carbondata.thrift
+++ b/format/src/main/thrift/carbondata.thrift
@@ -114,8 +114,12 @@ struct DataChunk{
}
/**
- * Represents a chunk of data. The chunk can be a single column stored in Column Major format or a group of columns stored in Row Major Format.
- * For V2 format.
+ * Represents the metadata of a data chunk.
+ * The chunk can be a single column stored in Column Major format or a group of columns stored
+ * in Row Major format.
+ *
+ * For V3, one data chunk is one page data of 32K rows.
+ * For V2 & V1, one data chunk is one blocklet data.
*/
struct DataChunk2{
1: required ChunkCompressionMeta chunk_meta; // The metadata of a chunk
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/emptyrow/TestEmptyRows.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/emptyrow/TestEmptyRows.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/emptyrow/TestEmptyRows.scala
index f31d434..5d6c07a 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/emptyrow/TestEmptyRows.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/emptyrow/TestEmptyRows.scala
@@ -28,7 +28,7 @@ class TestEmptyRows extends QueryTest with BeforeAndAfterAll {
override def beforeAll {
sql("drop table if exists emptyRowCarbonTable")
sql("drop table if exists emptyRowHiveTable")
- //eid,ename,sal,presal,comm,deptno,Desc
+
sql(
"create table if not exists emptyRowCarbonTable (eid int,ename String,sal decimal,presal " +
"decimal,comm decimal" +
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithNoMeasure.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithNoMeasure.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithNoMeasure.scala
index 3d85814..fa7b970 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithNoMeasure.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithNoMeasure.scala
@@ -28,7 +28,7 @@ import org.scalatest.BeforeAndAfterAll
class TestLoadDataWithNoMeasure extends QueryTest with BeforeAndAfterAll {
override def beforeAll {
- sql("DROP TABLE IF EXISTS nomeasureTest_sd")
+ sql("DROP TABLE IF EXISTS nomeasureTest")
sql(
"CREATE TABLE nomeasureTest (empno String, doj String) STORED BY 'org.apache.carbondata" +
".format'"
@@ -106,8 +106,8 @@ class TestLoadDataWithNoMeasure extends QueryTest with BeforeAndAfterAll {
}
override def afterAll {
- sql("drop table nomeasureTest")
- sql("drop table nomeasureTest_sd")
- sql("drop table nomeasureTest_scd")
+ sql("drop table if exists nomeasureTest")
+ sql("drop table if exists nomeasureTest_sd")
+ sql("drop table if exists nomeasureTest_scd")
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ColumnGroupDataTypesTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ColumnGroupDataTypesTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ColumnGroupDataTypesTestCase.scala
index f1c1d69..ab003c0 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ColumnGroupDataTypesTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ColumnGroupDataTypesTestCase.scala
@@ -27,6 +27,12 @@ import org.scalatest.BeforeAndAfterAll
class ColumnGroupDataTypesTestCase extends QueryTest with BeforeAndAfterAll {
override def beforeAll {
+ sql("drop table if exists colgrp")
+ sql("drop table if exists normal")
+ sql("drop table if exists colgrp_dictexclude_before")
+ sql("drop table if exists colgrp_dictexclude_after")
+ sql("drop table if exists colgrp_disorder")
+
sql("create table colgrp (column1 string,column2 string,column3 string,column4 string,column5 string,column6 string,column7 string,column8 string,column9 string,column10 string,measure1 int,measure2 int,measure3 int,measure4 int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES (\"COLUMN_GROUPS\"=\"(column2,column3,column4),(column7,column8,column9)\")")
sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/10dim_4msr.csv' INTO table colgrp options('FILEHEADER'='column1,column2,column3,column4,column5,column6,column7,column8,column9,column10,measure1,measure2,measure3,measure4')");
sql("create table normal (column1 string,column2 string,column3 string,column4 string,column5 string,column6 string,column7 string,column8 string,column9 string,column10 string,measure1 int,measure2 int,measure3 int,measure4 int) STORED BY 'org.apache.carbondata.format'")
[19/19] carbondata git commit: Merging 12-dev branch into master
branch. This closes #914
Posted by ra...@apache.org.
Merging 12-dev branch into master branch. This closes #914
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/46521add
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/46521add
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/46521add
Branch: refs/heads/master
Commit: 46521add0668ab2b8c7816690c5ceca092500dce
Parents: fb2441a fa08ab4
Author: ravipesala <ra...@gmail.com>
Authored: Mon May 15 10:34:56 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Mon May 15 10:34:56 2017 +0530
----------------------------------------------------------------------
.../dictionary/DictionaryCacheLoaderImpl.java | 3 +-
.../dictionary/DoubleArrayTrieDictionary.java | 5 +-
.../dictionary/ReverseDictionaryCache.java | 3 +-
.../core/compression/BigIntCompressor.java | 9 +-
.../core/compression/DoubleCompressor.java | 43 +-
.../core/compression/ValueCompressor.java | 3 +-
.../core/constants/CarbonCommonConstants.java | 13 +-
.../core/datastore/block/SegmentProperties.java | 42 +
.../block/TableBlockUniqueIdentifier.java | 3 +-
.../chunk/store/MeasureChunkStoreFactory.java | 26 +-
...feVariableLengthDimensionDataChunkStore.java | 32 +-
.../BlockIndexerStorageForNoInvertedIndex.java | 2 +-
...ndexerStorageForNoInvertedIndexForShort.java | 3 +-
.../compression/MeasureMetaDataModel.java | 8 +-
.../compression/ReaderCompressModel.java | 8 +-
.../compression/ValueCompressionHolder.java | 26 +-
.../compression/WriterCompressModel.java | 41 +-
.../compression/decimal/CompressByteArray.java | 4 +-
.../decimal/CompressionMaxMinByte.java | 6 +-
.../decimal/CompressionMaxMinDefault.java | 6 +-
.../decimal/CompressionMaxMinInt.java | 6 +-
.../decimal/CompressionMaxMinLong.java | 6 +-
.../decimal/CompressionMaxMinShort.java | 6 +-
.../nondecimal/CompressionNonDecimalByte.java | 6 +-
.../CompressionNonDecimalDefault.java | 6 +-
.../nondecimal/CompressionNonDecimalInt.java | 6 +-
.../nondecimal/CompressionNonDecimalLong.java | 6 +-
.../CompressionNonDecimalMaxMinByte.java | 6 +-
.../CompressionNonDecimalMaxMinDefault.java | 6 +-
.../CompressionNonDecimalMaxMinInt.java | 6 +-
.../CompressionNonDecimalMaxMinLong.java | 6 +-
.../CompressionNonDecimalMaxMinShort.java | 6 +-
.../nondecimal/CompressionNonDecimalShort.java | 6 +-
.../compression/none/CompressionNoneByte.java | 6 +-
.../none/CompressionNoneDefault.java | 6 +-
.../compression/none/CompressionNoneInt.java | 6 +-
.../compression/none/CompressionNoneLong.java | 6 +-
.../compression/none/CompressionNoneShort.java | 6 +-
.../dataholder/CarbonWriteDataHolder.java | 156 +--
.../core/datastore/impl/FileFactory.java | 10 +-
.../impl/btree/BTreeDataRefNodeFinder.java | 24 +-
.../HeavyCompressedDoubleArrayDataStore.java | 57 --
.../core/datastore/page/ColumnPage.java | 42 +
.../core/datastore/page/ComplexColumnPage.java | 77 ++
.../datastore/page/FixLengthColumnPage.java | 155 +++
.../datastore/page/VarLengthColumnPage.java | 42 +
.../datastore/page/compression/Compression.java | 23 +
.../datastore/page/encoding/ColumnCodec.java | 35 +
.../datastore/page/encoding/DummyCodec.java | 37 +
.../page/statistics/PageStatistics.java | 124 +++
.../page/statistics/StatisticsCollector.java | 66 ++
.../DateDirectDictionaryGenerator.java | 3 +-
.../core/keygenerator/mdkey/Bits.java | 9 +-
.../keygenerator/mdkey/NumberCompressor.java | 7 +-
.../core/metadata/ValueEncoderMeta.java | 18 +-
.../ThriftWrapperSchemaConverterImpl.java | 97 ++
.../core/metadata/datatype/DataType.java | 6 +-
.../core/metadata/schema/PartitionInfo.java | 91 ++
.../schema/partition/AbstractPartition.java | 42 +
.../schema/partition/HashPartition.java | 34 +
.../schema/partition/ListPartition.java | 36 +
.../schema/partition/PartitionType.java | 27 +
.../schema/partition/RangePartition.java | 40 +
.../core/metadata/schema/table/CarbonTable.java | 47 +
.../core/metadata/schema/table/TableSchema.java | 14 +
.../schema/table/column/ColumnSchema.java | 10 +
.../core/mutate/CarbonUpdateUtil.java | 14 +-
.../CarbonDictionaryMetadataReaderImpl.java | 12 +-
.../core/reader/CarbonIndexFileReader.java | 6 +-
.../impl/DictionaryBasedResultCollector.java | 12 +-
...structureBasedDictionaryResultCollector.java | 4 +-
.../core/scan/expression/ColumnExpression.java | 4 +-
.../core/scan/expression/LiteralExpression.java | 6 +-
.../scan/filter/FilterExpressionProcessor.java | 4 +-
.../carbondata/core/scan/filter/FilterUtil.java | 98 +-
.../executer/RowLevelFilterExecuterImpl.java | 9 +-
...velRangeLessThanEqualFilterExecuterImpl.java | 2 +-
.../RowLevelRangeLessThanFiterExecuterImpl.java | 2 +-
.../executer/ValueBasedFilterExecuterImpl.java | 6 +-
.../RowLevelRangeFilterResolverImpl.java | 5 +-
.../visitor/NoDictionaryTypeVisitor.java | 3 +-
.../visitor/RangeNoDictionaryTypeVisitor.java | 3 +-
.../core/scan/partition/HashPartitioner.java | 41 +
.../core/scan/partition/ListPartitioner.java | 65 ++
.../core/scan/partition/PartitionUtil.java | 109 +++
.../core/scan/partition/Partitioner.java | 27 +
.../core/scan/partition/RangePartitioner.java | 137 +++
.../AbstractDetailQueryResultIterator.java | 4 +-
.../iterator/DetailQueryResultIterator.java | 3 +-
.../scan/result/vector/CarbonColumnVector.java | 7 +
.../core/stats/PartitionStatistic.java | 59 ++
.../SegmentUpdateStatusManager.java | 5 +-
.../util/AbstractDataFileFooterConverter.java | 7 +
.../apache/carbondata/core/util/ByteUtil.java | 346 ++++++-
.../core/util/CarbonMetadataUtil.java | 44 +-
.../carbondata/core/util/CarbonProperties.java | 1 -
.../apache/carbondata/core/util/CarbonUtil.java | 35 +-
.../carbondata/core/util/CompressionFinder.java | 10 +-
.../carbondata/core/util/DataTypeUtil.java | 73 +-
.../apache/carbondata/core/util/NodeHolder.java | 31 +-
.../core/util/ValueCompressionUtil.java | 190 ++--
.../core/util/path/CarbonStorePath.java | 3 +-
.../core/util/path/CarbonTablePath.java | 6 +-
.../dictionary/AbstractDictionaryCacheTest.java | 6 +-
.../dictionary/ReverseDictionaryCacheTest.java | 4 +-
.../block/SegmentPropertiesTestUtil.java | 3 +-
.../filesystem/AlluxioCarbonFileTest.java | 6 +-
.../filesystem/HDFSCarbonFileTest.java | 9 +-
.../filesystem/LocalCarbonFileTest.java | 3 +-
.../filesystem/ViewFsCarbonFileTest.java | 6 +-
.../impl/btree/BTreeBlockFinderTest.java | 10 +-
.../ThriftWrapperSchemaConverterImplTest.java | 3 +-
.../core/scan/executor/util/QueryUtilTest.java | 9 +-
.../core/scan/filter/FilterUtilTest.java | 7 +-
.../core/util/CarbonMetadataUtilTest.java | 11 +-
.../core/util/RangeFilterProcessorTest.java | 21 +-
.../core/util/ValueCompressionUtilTest.java | 114 +--
.../core/writer/CarbonFooterWriterTest.java | 2 +-
.../src/main/resources/partition_data.csv | 25 +
.../examples/CarbonPartitionExample.scala | 131 +++
.../examples/CarbonSessionExample.scala | 1 +
format/src/main/thrift/carbondata.thrift | 8 +-
format/src/main/thrift/schema.thrift | 18 +
.../carbondata/hadoop/CarbonInputFormat.java | 4 +-
.../carbondata/hadoop/CarbonInputSplit.java | 6 +-
.../internal/index/impl/InMemoryBTreeIndex.java | 4 +-
.../impl/DictionaryDecodeReadSupport.java | 3 +
.../hadoop/util/BlockLevelTraverser.java | 4 +-
.../hadoop/test/util/StoreCreator.java | 5 +
.../testsuite/emptyrow/TestEmptyRows.scala | 2 +-
.../testsuite/dataload/TestLoadDataFrame.scala | 79 +-
.../dataload/TestLoadDataWithNoMeasure.scala | 8 +-
.../ColumnGroupDataTypesTestCase.scala | 6 +
.../partition/TestDDLForPartitionTable.scala | 127 +++
.../TestDataLoadingForPartitionTable.scala | 289 ++++++
.../testsuite/sortcolumns/TestSortColumns.scala | 274 ++++++
.../carbondata/spark/PartitionFactory.scala | 65 ++
.../spark/rdd/NewCarbonDataLoadRDD.scala | 79 ++
.../carbondata/spark/util/CommonUtil.scala | 37 +-
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 225 +++--
.../execution/command/carbonTableSchema.scala | 80 +-
.../spark/CarbonDataFrameWriter.scala | 8 +-
.../spark/rdd/CarbonDataRDDFactory.scala | 117 ++-
.../org/apache/spark/sql/CarbonSqlParser.scala | 38 +-
.../apache/spark/sql/TestCarbonSqlParser.scala | 56 +-
.../readsupport/SparkRowReadSupportImpl.java | 13 +-
.../vectorreader/ColumnarVectorWrapper.java | 13 +
.../spark/rdd/CarbonDataRDDFactory.scala | 126 ++-
.../spark/sql/CarbonDataFrameWriter.scala | 29 +-
.../org/apache/spark/sql/TableCreator.scala | 148 +--
.../spark/sql/parser/CarbonSparkSqlParser.scala | 39 +-
.../processing/merger/CarbonCompactionUtil.java | 3 +-
.../processing/merger/CarbonDataMergerUtil.java | 29 +-
.../merger/CompactionResultSortProcessor.java | 30 +-
.../merger/RowResultMergerProcessor.java | 14 +-
.../processing/model/CarbonLoadModel.java | 6 +-
.../newflow/CarbonDataLoadConfiguration.java | 24 +
.../newflow/DataLoadProcessBuilder.java | 36 +-
.../impl/NonDictionaryFieldConverterImpl.java | 21 +-
.../DictionaryServerClientDictionary.java | 3 +-
.../sort/impl/ParallelReadMergeSorterImpl.java | 5 +-
...arallelReadMergeSorterWithBucketingImpl.java | 7 +-
.../sort/unsafe/UnsafeCarbonRowPage.java | 142 +--
.../newflow/sort/unsafe/UnsafeSortDataRows.java | 27 +-
.../unsafe/comparator/UnsafeRowComparator.java | 10 +-
.../UnsafeRowComparatorForNormalDIms.java | 8 +-
.../holder/UnsafeFinalMergePageHolder.java | 4 +-
.../unsafe/holder/UnsafeInmemoryHolder.java | 5 +-
.../holder/UnsafeSortTempFileChunkHolder.java | 34 +-
.../merger/UnsafeIntermediateFileMerger.java | 38 +-
.../UnsafeSingleThreadFinalSortFilesMerger.java | 4 +-
.../CarbonRowDataWriterProcessorStepImpl.java | 319 +++++++
.../steps/DataConverterProcessorStepImpl.java | 3 +-
...ConverterProcessorWithBucketingStepImpl.java | 3 +-
.../newflow/steps/SortProcessorStepImpl.java | 3 +-
.../sortdata/IntermediateFileMerger.java | 38 +-
.../sortdata/NewRowComparator.java | 12 +-
.../sortdata/NewRowComparatorForNormalDims.java | 10 +-
.../sortandgroupby/sortdata/RowComparator.java | 10 +-
.../sortdata/RowComparatorForNormalDims.java | 10 +-
.../sortandgroupby/sortdata/SortDataRows.java | 42 +-
.../sortandgroupby/sortdata/SortParameters.java | 68 +-
.../sortdata/SortTempFileChunkHolder.java | 43 +-
.../store/CarbonDataFileAttributes.java | 10 +-
.../store/CarbonFactDataHandlerColumnar.java | 948 +++++++++----------
.../store/CarbonFactDataHandlerModel.java | 50 +-
.../store/SingleThreadFinalSortFilesMerger.java | 13 +-
.../store/colgroup/ColGroupBlockStorage.java | 29 +-
.../store/writer/AbstractFactDataWriter.java | 4 +-
.../writer/v3/CarbonFactDataWriterImplV3.java | 4 +-
.../util/CarbonDataProcessorUtil.java | 53 +-
.../processing/util/NonDictionaryUtil.java | 3 +-
.../store/colgroup/ColGroupMinMaxTest.java | 2 +
193 files changed, 5617 insertions(+), 1907 deletions(-)
----------------------------------------------------------------------
[16/19] carbondata git commit: [CARBONDATA-937] Data loading for
partition table(12-dev) This closes #842
Posted by ra...@apache.org.
[CARBONDATA-937] Data loading for partition table(12-dev) This closes #842
load data for partition table
fix comments
fix comments
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/b6b93f07
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/b6b93f07
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/b6b93f07
Branch: refs/heads/master
Commit: b6b93f076d53a4ba87d1f0a2123c04189f2a76e9
Parents: 86f6a81
Author: QiangCai <qi...@qq.com>
Authored: Mon Apr 24 11:28:28 2017 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Sun May 14 20:40:14 2017 +0800
----------------------------------------------------------------------
.../core/scan/partition/HashPartitioner.java | 41 +++
.../core/scan/partition/ListPartitioner.java | 65 +++++
.../core/scan/partition/PartitionUtil.java | 109 +++++++
.../core/scan/partition/Partitioner.java | 27 ++
.../core/scan/partition/RangePartitioner.java | 137 +++++++++
.../carbondata/hadoop/CarbonInputSplit.java | 6 +-
.../TestDataLoadingForPartitionTable.scala | 289 +++++++++++++++++++
.../carbondata/spark/PartitionFactory.scala | 65 +++++
.../spark/rdd/NewCarbonDataLoadRDD.scala | 79 +++++
.../spark/rdd/CarbonDataRDDFactory.scala | 117 +++++++-
.../spark/rdd/CarbonDataRDDFactory.scala | 126 +++++++-
11 files changed, 1039 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b6b93f07/core/src/main/java/org/apache/carbondata/core/scan/partition/HashPartitioner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/partition/HashPartitioner.java b/core/src/main/java/org/apache/carbondata/core/scan/partition/HashPartitioner.java
new file mode 100644
index 0000000..240c449
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/partition/HashPartitioner.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.scan.partition;
+
+/**
+ * Hash Partitioner
+ */
+public class HashPartitioner implements Partitioner {
+
+ private int numPartitions = 0;
+
+ public HashPartitioner(int numPartitions) {
+ this.numPartitions = numPartitions;
+ }
+
+ @Override public int numPartitions() {
+ return numPartitions;
+ }
+
+ @Override public int getPartition(Object key) {
+ if (key == null) {
+ return 0;
+ }
+ return (key.hashCode() & Integer.MAX_VALUE) % numPartitions;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b6b93f07/core/src/main/java/org/apache/carbondata/core/scan/partition/ListPartitioner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/partition/ListPartitioner.java b/core/src/main/java/org/apache/carbondata/core/scan/partition/ListPartitioner.java
new file mode 100644
index 0000000..bab2ede
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/partition/ListPartitioner.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.scan.partition;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.schema.PartitionInfo;
+
+/**
+ * List Partitioner
+ */
+public class ListPartitioner implements Partitioner {
+
+ /**
+ * map the value of ListPartition to partition id.
+ */
+ private Map<Object, Integer> map = new java.util.HashMap<Object, Integer>();
+
+ private int numPartitions;
+
+ public ListPartitioner(PartitionInfo partitionInfo) {
+ List<List<String>> values = partitionInfo.getListInfo();
+ DataType partitionColumnDataType = partitionInfo.getColumnSchemaList().get(0).getDataType();
+ numPartitions = values.size();
+ for (int i = 0; i < numPartitions; i++) {
+ for (String value : values.get(i)) {
+ map.put(PartitionUtil.getDataBasedOnDataType(value, partitionColumnDataType), i);
+ }
+ }
+ }
+
+ /**
+ * number of partitions
+ * add extra default partition
+ * @return
+ */
+ @Override public int numPartitions() {
+ return numPartitions + 1;
+ }
+
+ @Override public int getPartition(Object key) {
+ Integer partition = map.get(key);
+ if (partition == null) {
+ return numPartitions;
+ }
+ return partition;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b6b93f07/core/src/main/java/org/apache/carbondata/core/scan/partition/PartitionUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/partition/PartitionUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/partition/PartitionUtil.java
new file mode 100644
index 0000000..a5b3a9f
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/partition/PartitionUtil.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.scan.partition;
+
+import java.math.BigDecimal;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.BitSet;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.schema.PartitionInfo;
+import org.apache.carbondata.core.util.CarbonProperties;
+
+import org.apache.commons.lang.StringUtils;
+
+public class PartitionUtil {
+
+ private static LogService LOGGER = LogServiceFactory.getLogService(PartitionUtil.class.getName());
+
+ private static final ThreadLocal<DateFormat> timestampFormatter = new ThreadLocal<DateFormat>() {
+ @Override protected DateFormat initialValue() {
+ return new SimpleDateFormat(CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+ CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT));
+ }
+ };
+
+ private static final ThreadLocal<DateFormat> dateFormatter = new ThreadLocal<DateFormat>() {
+ @Override protected DateFormat initialValue() {
+ return new SimpleDateFormat(CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.CARBON_DATE_FORMAT,
+ CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT));
+ }
+ };
+
+ public static Partitioner getPartitioner(PartitionInfo partitionInfo) {
+ switch (partitionInfo.getPartitionType()) {
+ case HASH:
+ return new HashPartitioner(partitionInfo.getNumPartitions());
+ case LIST:
+ return new ListPartitioner(partitionInfo);
+ case RANGE:
+ return new RangePartitioner(partitionInfo);
+ default:
+ throw new UnsupportedOperationException(
+ "unsupport partition type: " + partitionInfo.getPartitionType().name());
+ }
+ }
+
+ public static Object getDataBasedOnDataType(String data, DataType actualDataType) {
+ if (data == null) {
+ return null;
+ }
+ if (actualDataType != DataType.STRING && StringUtils.isEmpty(data)) {
+ return null;
+ }
+ try {
+ switch (actualDataType) {
+ case STRING:
+ return data;
+ case INT:
+ return Integer.parseInt(data);
+ case SHORT:
+ return Short.parseShort(data);
+ case DOUBLE:
+ return Double.parseDouble(data);
+ case LONG:
+ return Long.parseLong(data);
+ case DATE:
+ return PartitionUtil.dateFormatter.get().parse(data).getTime();
+ case TIMESTAMP:
+ return PartitionUtil.timestampFormatter.get().parse(data).getTime();
+ case DECIMAL:
+ return new BigDecimal(data);
+ default:
+ return data;
+ }
+ } catch (Exception ex) {
+ return null;
+ }
+ }
+
+ public static BitSet generateBitSetBySize(int size, boolean isContainAll) {
+ BitSet bitSet = new BitSet(size);
+ if (isContainAll) {
+ bitSet.set(0, size);
+ }
+ return bitSet;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b6b93f07/core/src/main/java/org/apache/carbondata/core/scan/partition/Partitioner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/partition/Partitioner.java b/core/src/main/java/org/apache/carbondata/core/scan/partition/Partitioner.java
new file mode 100644
index 0000000..772a98e
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/partition/Partitioner.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.scan.partition;
+
+import java.io.Serializable;
+
+public interface Partitioner extends Serializable {
+
+ int numPartitions();
+
+ int getPartition(Object key);
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b6b93f07/core/src/main/java/org/apache/carbondata/core/scan/partition/RangePartitioner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/partition/RangePartitioner.java b/core/src/main/java/org/apache/carbondata/core/scan/partition/RangePartitioner.java
new file mode 100644
index 0000000..c73f0b5
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/partition/RangePartitioner.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.scan.partition;
+
+import java.io.Serializable;
+import java.math.BigDecimal;
+import java.util.List;
+
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.schema.PartitionInfo;
+import org.apache.carbondata.core.util.ByteUtil;
+
+/**
+ * Range Partitioner
+ */
+public class RangePartitioner implements Partitioner {
+
+ private int numPartitions;
+ private Object[] bounds;
+ private SerializableComparator comparator;
+
+ public RangePartitioner(PartitionInfo partitionInfo) {
+ List<String> values = partitionInfo.getRangeInfo();
+ DataType partitionColumnDataType = partitionInfo.getColumnSchemaList().get(0).getDataType();
+ numPartitions = values.size();
+ bounds = new Object[numPartitions];
+ if (partitionColumnDataType == DataType.STRING) {
+ for (int i = 0; i < numPartitions; i++) {
+ bounds[i] = ByteUtil.toBytes(values.get(i));
+ }
+ } else {
+ for (int i = 0; i < numPartitions; i++) {
+ bounds[i] = PartitionUtil.getDataBasedOnDataType(values.get(i), partitionColumnDataType);
+ }
+ }
+
+ switch (partitionColumnDataType) {
+ case INT:
+ comparator = new IntSerializableComparator();
+ break;
+ case SHORT:
+ comparator = new ShortSerializableComparator();
+ break;
+ case DOUBLE:
+ comparator = new DoubleSerializableComparator();
+ break;
+ case LONG:
+ case DATE:
+ case TIMESTAMP:
+ comparator = new LongSerializableComparator();
+ break;
+ case DECIMAL:
+ comparator = new BigDecimalSerializableComparator();
+ break;
+ default:
+ comparator = new ByteArraySerializableComparator();
+ }
+ }
+
+ /**
+ * number of partitions
+ * add extra default partition
+ * @return
+ */
+ @Override public int numPartitions() {
+ return numPartitions + 1;
+ }
+
+ @Override public int getPartition(Object key) {
+ if (key == null) {
+ return numPartitions;
+ } else {
+ for (int i = 0; i < numPartitions; i++) {
+ if (comparator.compareTo(key, bounds[i])) {
+ return i;
+ }
+ }
+ return numPartitions;
+ }
+ }
+
+ interface SerializableComparator extends Serializable {
+ boolean compareTo(Object key1, Object key2);
+ }
+
+ class ByteArraySerializableComparator implements SerializableComparator {
+ @Override public boolean compareTo(Object key1, Object key2) {
+ return ByteUtil.compare((byte[]) key1, (byte[]) key2) < 0;
+ }
+ }
+
+ class IntSerializableComparator implements SerializableComparator {
+ @Override public boolean compareTo(Object key1, Object key2) {
+ return (int) key1 - (int) key2 < 0;
+ }
+ }
+
+ class ShortSerializableComparator implements SerializableComparator {
+ @Override public boolean compareTo(Object key1, Object key2) {
+ return (short) key1 - (short) key2 < 0;
+ }
+ }
+
+ class DoubleSerializableComparator implements SerializableComparator {
+ @Override public boolean compareTo(Object key1, Object key2) {
+ return (double) key1 - (double) key2 < 0;
+ }
+ }
+
+ class LongSerializableComparator implements SerializableComparator {
+ @Override public boolean compareTo(Object key1, Object key2) {
+ return (long) key1 - (long) key2 < 0;
+ }
+ }
+
+ class BigDecimalSerializableComparator implements SerializableComparator {
+ @Override public boolean compareTo(Object key1, Object key2) {
+ return ((BigDecimal) key1).compareTo((BigDecimal) key2) < 0;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b6b93f07/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
index 0dcaba2..1a75bf9 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
@@ -237,8 +237,10 @@ public class CarbonInputSplit extends FileSplit
String filePath1 = this.getPath().getName();
String filePath2 = other.getPath().getName();
if (CarbonTablePath.isCarbonDataFile(filePath1)) {
- int firstTaskId = Integer.parseInt(CarbonTablePath.DataFileUtil.getTaskNo(filePath1));
- int otherTaskId = Integer.parseInt(CarbonTablePath.DataFileUtil.getTaskNo(filePath2));
+ int firstTaskId =
+ Integer.parseInt(CarbonTablePath.DataFileUtil.getTaskNo(filePath1).split("_")[0]);
+ int otherTaskId =
+ Integer.parseInt(CarbonTablePath.DataFileUtil.getTaskNo(filePath2).split("_")[0]);
if (firstTaskId != otherTaskId) {
return firstTaskId - otherTaskId;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b6b93f07/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDataLoadingForPartitionTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDataLoadingForPartitionTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDataLoadingForPartitionTable.scala
new file mode 100644
index 0000000..c0ee4f2
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDataLoadingForPartitionTable.scala
@@ -0,0 +1,289 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.spark.testsuite.partition
+
+import org.apache.spark.sql.common.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datastore.filesystem.{CarbonFile, CarbonFileFilter}
+import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.metadata.CarbonMetadata
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.core.util.path.CarbonTablePath
+
+class TestDataLoadingForPartitionTable extends QueryTest with BeforeAndAfterAll {
+
+ val defaultTimestampFormat = CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT)
+
+ override def beforeAll {
+ dropTable
+
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
+ sql(
+ """
+ | CREATE TABLE originTable (empno int, empname String, designation String, doj Timestamp,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int,salary int)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE originTable OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+
+ sql(
+ """
+ | CREATE TABLE originMultiLoads (empno int, empname String, designation String, doj Timestamp,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int,salary int)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE originMultiLoads OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE originMultiLoads OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE originMultiLoads OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ }
+
+ def validateDataFiles(tableUniqueName: String, sgementId: String, partitions: Seq[Int]): Unit = {
+ val carbonTable = CarbonMetadata.getInstance().getCarbonTable(tableUniqueName)
+ val tablePath = new CarbonTablePath(carbonTable.getStorePath, carbonTable.getDatabaseName,
+ carbonTable.getFactTableName)
+ val segmentDir = tablePath.getCarbonDataDirectoryPath("0", sgementId)
+ val carbonFile = FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir))
+ val dataFiles = carbonFile.listFiles(new CarbonFileFilter() {
+ override def accept(file: CarbonFile): Boolean = {
+ return file.getName.endsWith(".carbondata")
+ }
+ })
+
+ assert(dataFiles.size == partitions.size)
+
+ dataFiles.foreach { dataFile =>
+ val taskId = CarbonTablePath.DataFileUtil.getTaskNo(dataFile.getName).split("_")(0).toInt
+ assert(partitions.exists(_ == taskId))
+ }
+ }
+
+ test("data loading for partition table: hash partition") {
+ sql(
+ """
+ | CREATE TABLE hashTable (empname String, designation String, doj Timestamp,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (empno int)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('PARTITION_TYPE'='HASH','NUM_PARTITIONS'='3')
+ """.stripMargin)
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE hashTable OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+
+ validateDataFiles("default_hashTable", "0", Seq(0, 1, 2))
+
+ checkAnswer(sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from hashTable order by empno"),
+ sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from originTable order by empno"))
+
+ }
+
+ test("data loading for partition table: range partition") {
+ sql(
+ """
+ | CREATE TABLE rangeTable (empno int, empname String, designation String,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (doj Timestamp)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('PARTITION_TYPE'='RANGE',
+ | 'RANGE_INFO'='01-01-2010, 01-01-2015, 01-04-2015, 01-07-2015')
+ """.stripMargin)
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE rangeTable OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+
+ validateDataFiles("default_rangeTable", "0", Seq(0, 1, 3, 4))
+
+ checkAnswer(sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from rangeTable order by empno"),
+ sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from originTable order by empno"))
+
+ }
+
+ test("data loading for partition table: list partition") {
+ sql(
+ """
+ | CREATE TABLE listTable (empno int, empname String, designation String, doj Timestamp,
+ | workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (workgroupcategory int)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('PARTITION_TYPE'='LIST',
+ | 'LIST_INFO'='0, 1, (2, 3)')
+ """.stripMargin)
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE listTable OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+
+ validateDataFiles("default_listTable", "0", Seq(1, 2))
+
+ checkAnswer(sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from listTable order by empno"),
+ sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from originTable order by empno"))
+ }
+
+ test("Insert into for partition table: hash partition") {
+ sql(
+ """
+ | CREATE TABLE hashTableForInsert (empname String, designation String, doj Timestamp,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (empno int)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('PARTITION_TYPE'='HASH','NUM_PARTITIONS'='3')
+ """.stripMargin)
+ sql("insert into hashTableForInsert select empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary, empno from originTable")
+
+ validateDataFiles("default_hashTableForInsert", "0", Seq(0, 1, 2))
+
+ checkAnswer(sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from hashTableForInsert order by empno"),
+ sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from originTable order by empno"))
+
+ }
+
+ test("Insert into for partition table: range partition") {
+ sql(
+ """
+ | CREATE TABLE rangeTableForInsert (empno int, empname String, designation String,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (doj Timestamp)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('PARTITION_TYPE'='RANGE',
+ | 'RANGE_INFO'='01-01-2010, 01-01-2015, 01-04-2015, 01-07-2015')
+ """.stripMargin)
+ sql("insert into rangeTableForInsert select empno, empname, designation, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary, doj from originTable")
+
+ validateDataFiles("default_rangeTableForInsert", "0", Seq(0, 1, 3, 4))
+
+ checkAnswer(sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from rangeTableForInsert order by empno"),
+ sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from originTable order by empno"))
+
+ }
+
+ test("Insert into partition table: list partition") {
+ sql(
+ """
+ | CREATE TABLE listTableForInsert (empno int, empname String, designation String, doj Timestamp,
+ | workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (workgroupcategory int)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('PARTITION_TYPE'='LIST',
+ | 'LIST_INFO'='0, 1, (2, 3)')
+ """.stripMargin)
+ sql("insert into listTableForInsert select empno, empname, designation, doj, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary, workgroupcategory from originTable")
+
+ validateDataFiles("default_listTableForInsert", "0", Seq(1, 2))
+
+ checkAnswer(sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from listTableForInsert order by empno"),
+ sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from originTable order by empno"))
+ }
+
+ test("multiple data loading for partition table") {
+ sql(
+ """
+ | CREATE TABLE multiLoads (empname String, designation String, doj Timestamp,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (empno int)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('PARTITION_TYPE'='HASH','NUM_PARTITIONS'='3')
+ """.stripMargin)
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE multiLoads OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE multiLoads OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE multiLoads OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+
+ checkAnswer(sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from multiLoads order by empno"),
+ sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from originMultiLoads order by empno"))
+ }
+
+ test("multiple insertInto for partition table") {
+ sql(
+ """
+ | CREATE TABLE multiInserts (empname String, designation String, doj Timestamp,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (empno int)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('PARTITION_TYPE'='HASH','NUM_PARTITIONS'='3')
+ """.stripMargin)
+ sql("insert into multiInserts select empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary, empno from originTable")
+ sql("insert into multiInserts select empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary, empno from originTable")
+ sql("insert into multiInserts select empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary, empno from originTable")
+
+ checkAnswer(sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from multiInserts order by empno"),
+ sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from originMultiLoads order by empno"))
+ }
+
+ test("multiple data loading and insertInto for partition table") {
+ sql(
+ """
+ | CREATE TABLE loadAndInsert (empname String, designation String, doj Timestamp,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (empno int)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('PARTITION_TYPE'='HASH','NUM_PARTITIONS'='3')
+ """.stripMargin)
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE loadAndInsert OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ sql("insert into loadAndInsert select empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary, empno from originTable")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE loadAndInsert OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+
+ checkAnswer(sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from loadAndInsert order by empno"),
+ sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from originMultiLoads order by empno"))
+ }
+
+ override def afterAll = {
+ dropTable
+ if (defaultTimestampFormat == null) {
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+ CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
+ } else {
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, defaultTimestampFormat)
+ }
+ }
+
+ def dropTable = {
+ sql("drop table if exists originTable")
+ sql("drop table if exists hashTable")
+ sql("drop table if exists rangeTable")
+ sql("drop table if exists listTable")
+ sql("drop table if exists hashTableForInsert")
+ sql("drop table if exists rangeTableForInsert")
+ sql("drop table if exists listTableForInsert")
+ sql("drop table if exists originMultiLoads")
+ sql("drop table if exists multiLoads")
+ sql("drop table if exists multiInserts")
+ sql("drop table if exists loadAndInsert")
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b6b93f07/integration/spark-common/src/main/scala/org/apache/carbondata/spark/PartitionFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/PartitionFactory.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/PartitionFactory.scala
new file mode 100644
index 0000000..f7758a6
--- /dev/null
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/PartitionFactory.scala
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.spark
+
+import org.apache.spark.Partitioner
+
+import org.apache.carbondata.core.metadata.schema.PartitionInfo
+import org.apache.carbondata.core.metadata.schema.partition.PartitionType
+import org.apache.carbondata.core.scan.partition.{HashPartitioner => JavaHashPartitioner, ListPartitioner => JavaListPartitioner, RangePartitioner => JavaRangePartitioner}
+import org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException
+
+object PartitionFactory {
+
+ def getPartitioner(partitionInfo: PartitionInfo): Partitioner = {
+ partitionInfo.getPartitionType match {
+ case PartitionType.HASH => new HashPartitioner(partitionInfo.getNumPartitions)
+ case PartitionType.LIST => new ListPartitioner(partitionInfo)
+ case PartitionType.RANGE => new RangePartitioner(partitionInfo)
+ case partitionType =>
+ throw new CarbonDataLoadingException(s"Unsupport partition type: ${partitionType}")
+ }
+ }
+}
+
+class HashPartitioner(partitions: Int) extends Partitioner {
+
+ private val partitioner = new JavaHashPartitioner(partitions)
+
+ override def numPartitions: Int = partitioner.numPartitions()
+
+ override def getPartition(key: Any): Int = partitioner.getPartition(key)
+}
+
+class ListPartitioner(partitionInfo: PartitionInfo) extends Partitioner {
+
+ private val partitioner = new JavaListPartitioner(partitionInfo)
+
+ override def numPartitions: Int = partitioner.numPartitions()
+
+ override def getPartition(key: Any): Int = partitioner.getPartition(key)
+}
+
+class RangePartitioner(partitionInfo: PartitionInfo) extends Partitioner {
+
+ private val partitioner = new JavaRangePartitioner(partitionInfo)
+
+ override def numPartitions: Int = partitioner.numPartitions()
+
+ override def getPartition(key: Any): Int = partitioner.getPartition(key)
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b6b93f07/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
index a6d231d..4d0770d 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
@@ -566,3 +566,82 @@ class LazyRddIterator(serializer: SerializerInstance,
}
}
+
+/*
+ * It loads the data to carbon from RDD for partition table
+ * @see org.apache.carbondata.processing.newflow.DataLoadExecutor
+ */
+class PartitionTableDataLoaderRDD[K, V](
+ sc: SparkContext,
+ result: DataLoadResult[K, V],
+ carbonLoadModel: CarbonLoadModel,
+ loadCount: Integer,
+ tableCreationTime: Long,
+ schemaLastUpdatedTime: Long,
+ prev: RDD[Row]) extends RDD[(K, V)](prev) {
+
+
+ override def compute(theSplit: Partition, context: TaskContext): Iterator[(K, V)] = {
+ val LOGGER = LogServiceFactory.getLogService(this.getClass.getName)
+ val iter = new Iterator[(K, V)] {
+ val partitionID = "0"
+ val loadMetadataDetails = new LoadMetadataDetails()
+ val model: CarbonLoadModel = carbonLoadModel
+ val uniqueLoadStatusId =
+ carbonLoadModel.getTableName + CarbonCommonConstants.UNDERSCORE + theSplit.index
+ try {
+
+ loadMetadataDetails.setPartitionCount(partitionID)
+ loadMetadataDetails.setLoadStatus(CarbonCommonConstants.STORE_LOADSTATUS_SUCCESS)
+ carbonLoadModel.setPartitionId(partitionID)
+ carbonLoadModel.setSegmentId(String.valueOf(loadCount))
+ carbonLoadModel.setTaskNo(String.valueOf(theSplit.index))
+ carbonLoadModel.setPreFetch(false)
+
+ val recordReaders = Array[CarbonIterator[Array[AnyRef]]] {
+ new NewRddIterator(firstParent[Row].iterator(theSplit, context), carbonLoadModel, context)
+ }
+
+ val loader = new SparkPartitionLoader(model,
+ theSplit.index,
+ null,
+ String.valueOf(loadCount),
+ loadMetadataDetails)
+ // Intialize to set carbon properties
+ loader.initialize()
+ new DataLoadExecutor().execute(model, loader.storeLocation, recordReaders)
+ } catch {
+ case e: BadRecordFoundException =>
+ loadMetadataDetails.setLoadStatus(CarbonCommonConstants.STORE_LOADSTATUS_PARTIAL_SUCCESS)
+ logInfo("Bad Record Found")
+ case e: Exception =>
+ loadMetadataDetails.setLoadStatus(CarbonCommonConstants.STORE_LOADSTATUS_FAILURE)
+ logInfo("DataLoad For Partition Table failure", e)
+ LOGGER.error(e)
+ throw e
+ } finally {
+ // clean up the folders and files created locally for data load operation
+ CarbonLoaderUtil.deleteLocalDataLoadFolderLocation(model, false)
+ // in case of failure the same operation will be re-tried several times.
+ // So print the data load statistics only in case of non failure case
+ if (!CarbonCommonConstants.STORE_LOADSTATUS_FAILURE
+ .equals(loadMetadataDetails.getLoadStatus)) {
+ CarbonTimeStatisticsFactory.getLoadStatisticsInstance
+ .printStatisticsInfo(model.getPartitionId)
+ }
+ }
+ var finished = false
+
+ override def hasNext: Boolean = !finished
+
+ override def next(): (K, V) = {
+ finished = true
+ result.getKey(uniqueLoadStatusId, loadMetadataDetails)
+ }
+ }
+ iter
+ }
+
+ override protected def getPartitions: Array[Partition] = firstParent[Row].partitions
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b6b93f07/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index 8f4727a..9685871 100644
--- a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -17,6 +17,7 @@
package org.apache.carbondata.spark.rdd
+import java.text.SimpleDateFormat
import java.util
import java.util.UUID
import java.util.concurrent._
@@ -28,10 +29,11 @@ import scala.util.control.Breaks._
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
+import org.apache.hadoop.io.NullWritable
import org.apache.hadoop.mapreduce.Job
import org.apache.hadoop.mapreduce.lib.input.{FileInputFormat, FileSplit}
import org.apache.spark.{SparkEnv, SparkException}
-import org.apache.spark.rdd.{DataLoadCoalescedRDD, DataLoadPartitionCoalescer, UpdateCoalescedRDD}
+import org.apache.spark.rdd.{DataLoadCoalescedRDD, DataLoadPartitionCoalescer, NewHadoopRDD, RDD, UpdateCoalescedRDD}
import org.apache.spark.sql.{CarbonEnv, DataFrame, Row, SQLContext}
import org.apache.spark.sql.execution.command.{AlterTableModel, CompactionModel, ExecutionErrors, UpdateTableModel}
import org.apache.spark.sql.hive.DistributionUtil
@@ -43,12 +45,15 @@ import org.apache.carbondata.core.datastore.block.{Distributable, TableBlockInfo
import org.apache.carbondata.core.dictionary.server.DictionaryServer
import org.apache.carbondata.core.locks.{CarbonLockFactory, ICarbonLock, LockUsage}
import org.apache.carbondata.core.metadata.{CarbonTableIdentifier, ColumnarFormatVersion}
+import org.apache.carbondata.core.metadata.datatype.DataType
+import org.apache.carbondata.core.metadata.schema.partition.PartitionType
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.mutate.CarbonUpdateUtil
+import org.apache.carbondata.core.scan.partition.PartitionUtil
import org.apache.carbondata.core.statusmanager.LoadMetadataDetails
-import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.core.util.{ByteUtil, CarbonProperties}
import org.apache.carbondata.core.util.path.CarbonStorePath
-import org.apache.carbondata.processing.csvload.BlockDetails
+import org.apache.carbondata.processing.csvload.{BlockDetails, CSVInputFormat, StringArrayWritable}
import org.apache.carbondata.processing.etl.DataLoadingException
import org.apache.carbondata.processing.merger.{CarbonCompactionUtil, CarbonDataMergerUtil, CompactionType}
import org.apache.carbondata.processing.model.CarbonLoadModel
@@ -56,7 +61,7 @@ import org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingExcep
import org.apache.carbondata.spark._
import org.apache.carbondata.spark.load._
import org.apache.carbondata.spark.splits.TableSplit
-import org.apache.carbondata.spark.util.{CarbonQueryUtil, CommonUtil}
+import org.apache.carbondata.spark.util.{CarbonQueryUtil, CarbonScalaUtil, CommonUtil}
/**
* This is the factory class which can create different RDD depends on user needs.
@@ -732,6 +737,23 @@ object CarbonDataRDDFactory {
}
+ def loadDataForPartitionTable(): Unit = {
+ try {
+ val rdd = repartitionInputData(sqlContext, dataFrame, carbonLoadModel)
+ status = new PartitionTableDataLoaderRDD(sqlContext.sparkContext,
+ new DataLoadResultImpl(),
+ carbonLoadModel,
+ currentLoadCount,
+ tableCreationTime,
+ schemaLastUpdatedTime,
+ rdd).collect()
+ } catch {
+ case ex: Exception =>
+ LOGGER.error(ex, "load data failed for partition table")
+ throw ex
+ }
+ }
+
if (!updateModel.isDefined) {
CarbonLoaderUtil.checkAndCreateCarbonDataLocation(storePath,
carbonLoadModel.getDatabaseName, carbonLoadModel.getTableName, currentLoadCount.toString)
@@ -742,10 +764,11 @@ object CarbonDataRDDFactory {
try {
if (updateModel.isDefined) {
loadDataFrameForUpdate()
+ } else if (carbonTable.getPartitionInfo(carbonTable.getFactTableName) != null) {
+ loadDataForPartitionTable()
} else if (dataFrame.isDefined) {
loadDataFrame()
- }
- else {
+ } else {
loadDataFile()
}
if (updateModel.isDefined) {
@@ -932,8 +955,86 @@ object CarbonDataRDDFactory {
}
- private def writeDictionary(carbonLoadModel: CarbonLoadModel,
- result: Option[DictionaryServer], writeAll: Boolean) = {
+ /**
+ * repartition the input data for partiton table.
+ * @param sqlContext
+ * @param dataFrame
+ * @param carbonLoadModel
+ * @return
+ */
+ private def repartitionInputData(sqlContext: SQLContext,
+ dataFrame: Option[DataFrame],
+ carbonLoadModel: CarbonLoadModel): RDD[Row] = {
+ val carbonTable = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
+ val partitionInfo = carbonTable.getPartitionInfo(carbonTable.getFactTableName)
+ val partitionColumn = partitionInfo.getColumnSchemaList.get(0).getColumnName
+ val partitionColumnDataType = partitionInfo.getColumnSchemaList.get(0).getDataType
+ val columns = carbonLoadModel.getCsvHeaderColumns
+ var partitionColumnIndex = -1
+ for (i <- 0 until columns.length) {
+ if (partitionColumn.equals(columns(i))) {
+ partitionColumnIndex = i
+ }
+ }
+ if (partitionColumnIndex == -1) {
+ throw new DataLoadingException("Partition column not found.")
+ }
+ // generate RDD[(K, V)] to use the partitionBy method of PairRDDFunctions
+ val inputRDD: RDD[(String, Row)] = if (dataFrame.isDefined) {
+ // input data from DataFrame
+ val timestampFormatString = CarbonProperties.getInstance().getProperty(CarbonCommonConstants
+ .CARBON_TIMESTAMP_FORMAT, CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
+ val timeStampFormat = new SimpleDateFormat(timestampFormatString)
+ val dateFormatString = CarbonProperties.getInstance().getProperty(CarbonCommonConstants
+ .CARBON_DATE_FORMAT, CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT)
+ val dateFormat = new SimpleDateFormat(dateFormatString)
+ val delimiterLevel1 = carbonLoadModel.getComplexDelimiterLevel1
+ val delimiterLevel2 = carbonLoadModel.getComplexDelimiterLevel2
+ val serializationNullFormat =
+ carbonLoadModel.getSerializationNullFormat.split(CarbonCommonConstants.COMMA, 2)(1)
+ dataFrame.get.rdd.map { row =>
+ (CarbonScalaUtil.getString(row.get(partitionColumnIndex), serializationNullFormat,
+ delimiterLevel1, delimiterLevel2, timeStampFormat, dateFormat), row)
+ }
+ } else {
+ // input data from csv files
+ val hadoopConfiguration = new Configuration()
+ CommonUtil.configureCSVInputFormat(hadoopConfiguration, carbonLoadModel)
+ hadoopConfiguration.set(FileInputFormat.INPUT_DIR, carbonLoadModel.getFactFilePath)
+ val columnCount = columns.length
+ new NewHadoopRDD[NullWritable, StringArrayWritable](
+ sqlContext.sparkContext,
+ classOf[CSVInputFormat],
+ classOf[NullWritable],
+ classOf[StringArrayWritable],
+ hadoopConfiguration
+ ).map { currentRow =>
+ val row = new StringArrayRow(new Array[String](columnCount))
+ (currentRow._2.get()(partitionColumnIndex), row.setValues(currentRow._2.get()))
+ }
+ }
+
+ val partitioner = PartitionFactory.getPartitioner(partitionInfo)
+ if (partitionColumnDataType == DataType.STRING) {
+ if (partitionInfo.getPartitionType == PartitionType.RANGE) {
+ inputRDD.map { row => (ByteUtil.toBytes(row._1), row._2) }
+ .partitionBy(partitioner)
+ .map(_._2)
+ } else {
+ inputRDD.partitionBy(partitioner)
+ .map(_._2)
+ }
+ } else {
+ inputRDD.map { row =>
+ (PartitionUtil.getDataBasedOnDataType(row._1, partitionColumnDataType), row._2)
+ }
+ .partitionBy(partitioner)
+ .map(_._2)
+ }
+ }
+
+ private def writeDictionary(carbonLoadModel: CarbonLoadModel,
+ result: Option[DictionaryServer], writeAll: Boolean) = {
// write dictionary file and shutdown dictionary server
val uniqueTableName: String = s"${ carbonLoadModel.getDatabaseName }_${
carbonLoadModel.getTableName }"
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b6b93f07/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index 835af35..812151e 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -17,8 +17,8 @@
package org.apache.carbondata.spark.rdd
+import java.text.SimpleDateFormat
import java.util
-import java.util.UUID
import java.util.concurrent._
import scala.collection.JavaConverters._
@@ -28,10 +28,11 @@ import scala.util.control.Breaks._
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
+import org.apache.hadoop.io.NullWritable
import org.apache.hadoop.mapreduce.Job
import org.apache.hadoop.mapreduce.lib.input.{FileInputFormat, FileSplit}
import org.apache.spark.{SparkEnv, SparkException}
-import org.apache.spark.rdd.{DataLoadCoalescedRDD, DataLoadPartitionCoalescer, UpdateCoalescedRDD}
+import org.apache.spark.rdd.{DataLoadCoalescedRDD, DataLoadPartitionCoalescer, NewHadoopRDD, RDD}
import org.apache.spark.sql.{CarbonEnv, DataFrame, Row, SQLContext}
import org.apache.spark.sql.execution.command.{AlterTableModel, CompactionModel, ExecutionErrors, UpdateTableModel}
import org.apache.spark.sql.hive.DistributionUtil
@@ -43,20 +44,22 @@ import org.apache.carbondata.core.datastore.block.{Distributable, TableBlockInfo
import org.apache.carbondata.core.dictionary.server.DictionaryServer
import org.apache.carbondata.core.locks.{CarbonLockFactory, ICarbonLock, LockUsage}
import org.apache.carbondata.core.metadata.{CarbonTableIdentifier, ColumnarFormatVersion}
+import org.apache.carbondata.core.metadata.datatype.DataType
+import org.apache.carbondata.core.metadata.schema.partition.PartitionType
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.mutate.CarbonUpdateUtil
+import org.apache.carbondata.core.scan.partition.PartitionUtil
import org.apache.carbondata.core.statusmanager.LoadMetadataDetails
-import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.core.util.path.CarbonStorePath
-import org.apache.carbondata.processing.csvload.BlockDetails
+import org.apache.carbondata.core.util.{ByteUtil, CarbonProperties}
+import org.apache.carbondata.processing.csvload.{BlockDetails, CSVInputFormat, StringArrayWritable}
import org.apache.carbondata.processing.etl.DataLoadingException
import org.apache.carbondata.processing.merger.{CarbonCompactionUtil, CarbonDataMergerUtil, CompactionType}
import org.apache.carbondata.processing.model.CarbonLoadModel
import org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException
-import org.apache.carbondata.spark._
+import org.apache.carbondata.spark.{DataLoadResultImpl, PartitionFactory}
import org.apache.carbondata.spark.load._
import org.apache.carbondata.spark.splits.TableSplit
-import org.apache.carbondata.spark.util.{CarbonQueryUtil, CommonUtil}
+import org.apache.carbondata.spark.util.{CarbonQueryUtil, CarbonScalaUtil, CommonUtil}
/**
* This is the factory class which can create different RDD depends on user needs.
@@ -653,6 +656,23 @@ object CarbonDataRDDFactory {
}
}
+ def loadDataForPartitionTable(): Unit = {
+ try {
+ val rdd = repartitionInputData(sqlContext, dataFrame, carbonLoadModel)
+ status = new PartitionTableDataLoaderRDD(sqlContext.sparkContext,
+ new DataLoadResultImpl(),
+ carbonLoadModel,
+ currentLoadCount,
+ tableCreationTime,
+ schemaLastUpdatedTime,
+ rdd).collect()
+ } catch {
+ case ex: Exception =>
+ LOGGER.error(ex, "load data failed for partition table")
+ throw ex
+ }
+ }
+
if (!updateModel.isDefined) {
CarbonLoaderUtil.checkAndCreateCarbonDataLocation(storePath,
carbonLoadModel.getDatabaseName, carbonLoadModel.getTableName, currentLoadCount.toString)
@@ -661,11 +681,15 @@ object CarbonDataRDDFactory {
var errorMessage: String = "DataLoad failure"
var executorMessage: String = ""
try {
- if (dataFrame.isDefined) {
- loadDataFrame()
- }
- else {
- loadDataFile()
+ if (carbonTable.getPartitionInfo(carbonTable.getFactTableName) != null) {
+ loadDataForPartitionTable()
+ } else {
+ if (dataFrame.isDefined) {
+ loadDataFrame()
+ }
+ else {
+ loadDataFile()
+ }
}
if (updateModel.isDefined) {
@@ -859,6 +883,84 @@ object CarbonDataRDDFactory {
}
+ /**
+ * repartition the input data for partiton table.
+ * @param sqlContext
+ * @param dataFrame
+ * @param carbonLoadModel
+ * @return
+ */
+ private def repartitionInputData(sqlContext: SQLContext,
+ dataFrame: Option[DataFrame],
+ carbonLoadModel: CarbonLoadModel): RDD[Row] = {
+ val carbonTable = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
+ val partitionInfo = carbonTable.getPartitionInfo(carbonTable.getFactTableName)
+ val partitionColumn = partitionInfo.getColumnSchemaList.get(0).getColumnName
+ val partitionColumnDataType = partitionInfo.getColumnSchemaList.get(0).getDataType
+ val columns = carbonLoadModel.getCsvHeaderColumns
+ var partitionColumnIndex = -1
+ for (i <- 0 until columns.length) {
+ if (partitionColumn.equals(columns(i))) {
+ partitionColumnIndex = i
+ }
+ }
+ if (partitionColumnIndex == -1) {
+ throw new DataLoadingException("Partition column not found.")
+ }
+ // generate RDD[(K, V)] to use the partitionBy method of PairRDDFunctions
+ val inputRDD: RDD[(String, Row)] = if (dataFrame.isDefined) {
+ // input data from DataFrame
+ val timestampFormatString = CarbonProperties.getInstance().getProperty(CarbonCommonConstants
+ .CARBON_TIMESTAMP_FORMAT, CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
+ val timeStampFormat = new SimpleDateFormat(timestampFormatString)
+ val dateFormatString = CarbonProperties.getInstance().getProperty(CarbonCommonConstants
+ .CARBON_DATE_FORMAT, CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT)
+ val dateFormat = new SimpleDateFormat(dateFormatString)
+ val delimiterLevel1 = carbonLoadModel.getComplexDelimiterLevel1
+ val delimiterLevel2 = carbonLoadModel.getComplexDelimiterLevel2
+ val serializationNullFormat =
+ carbonLoadModel.getSerializationNullFormat.split(CarbonCommonConstants.COMMA, 2)(1)
+ dataFrame.get.rdd.map { row =>
+ (CarbonScalaUtil.getString(row.get(partitionColumnIndex), serializationNullFormat,
+ delimiterLevel1, delimiterLevel2, timeStampFormat, dateFormat), row)
+ }
+ } else {
+ // input data from csv files
+ val hadoopConfiguration = new Configuration()
+ CommonUtil.configureCSVInputFormat(hadoopConfiguration, carbonLoadModel)
+ hadoopConfiguration.set(FileInputFormat.INPUT_DIR, carbonLoadModel.getFactFilePath)
+ val columnCount = columns.length
+ new NewHadoopRDD[NullWritable, StringArrayWritable](
+ sqlContext.sparkContext,
+ classOf[CSVInputFormat],
+ classOf[NullWritable],
+ classOf[StringArrayWritable],
+ hadoopConfiguration
+ ).map { currentRow =>
+ val row = new StringArrayRow(new Array[String](columnCount))
+ (currentRow._2.get()(partitionColumnIndex), row.setValues(currentRow._2.get()))
+ }
+ }
+
+ val partitioner = PartitionFactory.getPartitioner(partitionInfo)
+ if (partitionColumnDataType == DataType.STRING) {
+ if (partitionInfo.getPartitionType == PartitionType.RANGE) {
+ inputRDD.map { row => (ByteUtil.toBytes(row._1), row._2) }
+ .partitionBy(partitioner)
+ .map(_._2)
+ } else {
+ inputRDD.partitionBy(partitioner)
+ .map(_._2)
+ }
+ } else {
+ inputRDD.map { row =>
+ (PartitionUtil.getDataBasedOnDataType(row._1, partitionColumnDataType), row._2)
+ }
+ .partitionBy(partitioner)
+ .map(_._2)
+ }
+ }
+
private def writeDictionary(carbonLoadModel: CarbonLoadModel,
result: Option[DictionaryServer], writeAll: Boolean) = {
// write dictionary file and shutdown dictionary server
[11/19] carbondata git commit: [CARBONDATA-936] Parse partition table
ddl This closes #882
Posted by ra...@apache.org.
[CARBONDATA-936] Parse partition table ddl This closes #882
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/dae342bd
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/dae342bd
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/dae342bd
Branch: refs/heads/master
Commit: dae342bd57fe6d4704ad754b58438c1ec9d78516
Parents: f4d081e
Author: lionelcao <wh...@gmail.com>
Authored: Fri May 12 21:52:17 2017 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Sun May 14 20:40:13 2017 +0800
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 6 +-
.../core/metadata/schema/PartitionInfo.java | 29 ++--
.../src/main/resources/partition_data.csv | 25 ++++
.../examples/CarbonPartitionExample.scala | 131 +++++++++++++++++++
format/src/main/thrift/schema.thrift | 2 +-
.../carbondata/spark/util/CommonUtil.scala | 37 +++++-
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 100 +++++++++-----
.../execution/command/carbonTableSchema.scala | 5 +-
.../org/apache/spark/sql/TableCreator.scala | 5 +-
.../spark/sql/parser/CarbonSparkSqlParser.scala | 37 +++---
10 files changed, 300 insertions(+), 77 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dae342bd/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index b4235e2..33b0ef7 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -816,8 +816,10 @@ public final class CarbonCommonConstants {
public static final String DICTIONARY_EXCLUDE = "dictionary_exclude";
public static final String DICTIONARY_INCLUDE = "dictionary_include";
public static final String SORT_COLUMNS = "sort_columns";
- public static final String PARTITIONCLASS = "partitionclass";
- public static final String PARTITIONCOUNT = "partitioncount";
+ public static final String PARTITION_TYPE = "partition_type";
+ public static final String NUM_PARTITIONS = "num_partitions";
+ public static final String RANGE_INFO = "range_info";
+ public static final String LIST_INFO = "list_info";
public static final String COLUMN_PROPERTIES = "columnproperties";
// table block size in MB
public static final String TABLE_BLOCKSIZE = "table_blocksize";
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dae342bd/core/src/main/java/org/apache/carbondata/core/metadata/schema/PartitionInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/PartitionInfo.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/PartitionInfo.java
index 86ef3c5..cd4ac0e 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/PartitionInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/PartitionInfo.java
@@ -24,40 +24,29 @@ import org.apache.carbondata.core.metadata.schema.partition.PartitionType;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
/**
- * Partition Information of carbon partition table
+ * Partition information of carbon partition table
*/
public class PartitionInfo implements Serializable {
- /**
- * Partition columns
- */
private List<ColumnSchema> columnSchemaList;
- /**
- * partition type
- */
private PartitionType partitionType;
/**
- * Range Partition definition
+ * range information defined for range partition table
*/
private List<String> rangeInfo;
/**
- * List Partition definition
+ * value list defined for list partition table
*/
private List<List<String>> listInfo;
/**
- * Hash Partition numbers
+ * number of partitions
*/
- private int hashNumber;
+ private int numPartitions;
- /**
- * For range partition table
- * @param columnSchemaList
- * @param partitionType
- */
public PartitionInfo(List<ColumnSchema> columnSchemaList, PartitionType partitionType) {
this.columnSchemaList = columnSchemaList;
this.partitionType = partitionType;
@@ -71,12 +60,12 @@ public class PartitionInfo implements Serializable {
return partitionType;
}
- public void setHashNumber(int hashNumber) {
- this.hashNumber = hashNumber;
+ public void setNumPartitions(int numPartitions) {
+ this.numPartitions = numPartitions;
}
- public int getHashNumber() {
- return hashNumber;
+ public int getNumPartitions() {
+ return numPartitions;
}
public void setRangeInfo(List<String> rangeInfo) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dae342bd/examples/spark2/src/main/resources/partition_data.csv
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/resources/partition_data.csv b/examples/spark2/src/main/resources/partition_data.csv
new file mode 100644
index 0000000..2d521b7
--- /dev/null
+++ b/examples/spark2/src/main/resources/partition_data.csv
@@ -0,0 +1,25 @@
+vin,logdate,phonenumber,country,area
+A42158424831,2016/02/12,125371341,China,Asia
+A42158473831,2016/01/12,125371342,China,Asia
+A42152474832,2016/02/12,125371343,US,America
+A42151477823,2016/12/12,125371344,China,Asia
+A42158474135,2016/02/15,125371345,Japan,Asia
+A42258434831,2016/12/12,125371346,China,Asia
+A42158475831,2016/05/12,125371347,UK,Europe
+A41158494830,2015/07/12,225371348,China,Asia
+A42158424831,2015/02/12,225371349,China,Asia
+A42158473830,2014/01/12,225371310,China,Asia
+A42152474830,2013/02/12,325371311,US,America
+A42151477823,2012/12/12,425371312,China,Asia
+A42158474133,2012/02/15,325371313,Japan,Asia
+A42258434835,2013/12/12,525371314,China,Asia
+A42158475836,2014/05/12,625371315,UK,Europe
+A41158494838,2015/07/12,525371316,China,Asia
+A42158424833,2016/02/12,425371317,China,Asia
+A42158473832,2017/01/12,325371318,China,Asia
+A42152474834,2011/02/12,225371319,US,America
+A42151477824,2012/12/12,225371320,China,Asia
+A42158474137,2013/02/15,325371321,Japan,Asia
+A42258434837,2014/12/12,25371322,China,Asia
+A42158475838,2014/05/12,425371323,UK,Europe
+A41158494839,2016/07/12,625371324,China,Asia
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dae342bd/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonPartitionExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonPartitionExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonPartitionExample.scala
new file mode 100644
index 0000000..5f07b4b
--- /dev/null
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonPartitionExample.scala
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.examples
+
+import java.io.File
+
+import org.apache.spark.sql.SparkSession
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+
+object CarbonPartitionExample {
+
+ def main(args: Array[String]) {
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+ val storeLocation = s"$rootPath/examples/spark2/target/store"
+ val warehouse = s"$rootPath/examples/spark2/target/warehouse"
+ val metastoredb = s"$rootPath/examples/spark2/target"
+ val testData = s"$rootPath/examples/spark2/src/main/resources/partition_data.csv"
+
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
+
+ import org.apache.spark.sql.CarbonSession._
+
+ val spark = SparkSession
+ .builder()
+ .master("local")
+ .appName("CarbonPartitionExample")
+ .config("spark.sql.warehouse.dir", warehouse)
+ .getOrCreateCarbonSession(storeLocation, metastoredb)
+
+ spark.sparkContext.setLogLevel("WARN")
+
+ // none partition table
+ spark.sql("DROP TABLE IF EXISTS t0")
+
+ spark.sql("""
+ | CREATE TABLE IF NOT EXISTS t0
+ | (
+ | vin String,
+ | logdate Timestamp,
+ | phonenumber Long,
+ | country String,
+ | area String
+ | )
+ | STORED BY 'carbondata'
+ """.stripMargin)
+
+ // range partition
+ spark.sql("DROP TABLE IF EXISTS t1")
+
+ spark.sql("""
+ | CREATE TABLE IF NOT EXISTS t1
+ | (
+ | vin String,
+ | logdate Timestamp,
+ | phonenumber Long,
+ | country String,
+ | area String
+ | )
+ | PARTITIONED BY (logdate Timestamp)
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('PARTITION_TYPE'='RANGE',
+ | 'RANGE_INFO'='20140101, 2015/01/01 ,2016-01-01, ')
+ """.stripMargin)
+
+ // hash partition
+ spark.sql("DROP TABLE IF EXISTS t3")
+
+ spark.sql("""
+ | CREATE TABLE IF NOT EXISTS t3
+ | (
+ | vin String,
+ | logdate Timestamp,
+ | phonenumber Long,
+ | country String,
+ | area String
+ | )
+ | PARTITIONED BY (vin String)
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('PARTITION_TYPE'='HASH','NUM_PARTITIONS'='5')
+ """.stripMargin)
+
+ // list partition
+ spark.sql("DROP TABLE IF EXISTS t5")
+
+ spark.sql("""
+ | CREATE TABLE IF NOT EXISTS t5
+ | (
+ | vin String,
+ | logdate Timestamp,
+ | phonenumber Long,
+ | country String,
+ | area String
+ |)
+ | PARTITIONED BY (country string)
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('PARTITION_TYPE'='LIST',
+ | 'LIST_INFO'='(China,United States),UK ,japan,(Canada,Russia), South Korea ')
+ """.stripMargin)
+
+ // spark.sql(s"""
+ // LOAD DATA LOCAL INPATH '$testData' into table t3
+ // options('BAD_RECORDS_ACTION'='FORCE')
+ // """)
+
+ // spark.sql("select vin, count(*) from t3 group by vin
+ // order by count(*) desc").show(50)
+
+ // Drop table
+ // spark.sql("DROP TABLE IF EXISTS t3")
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dae342bd/format/src/main/thrift/schema.thrift
----------------------------------------------------------------------
diff --git a/format/src/main/thrift/schema.thrift b/format/src/main/thrift/schema.thrift
index 695333c..3385245 100644
--- a/format/src/main/thrift/schema.thrift
+++ b/format/src/main/thrift/schema.thrift
@@ -132,7 +132,7 @@ struct SchemaEvolution{
struct PartitionInfo{
1: required list<ColumnSchema> partition_columns;
2: required PartitionType partition_type;
- 3: optional i32 hash_number; // number of partitions defined in hash partition table
+ 3: optional i32 num_partitions; // number of partitions defined in hash partition table
4: optional list<list<string>> list_info; // value list of list partition table
5: optional list<string> range_info; // range value list of range partition table
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dae342bd/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
index 679a4e7..5314a15 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
@@ -26,7 +26,8 @@ import org.apache.commons.lang3.StringUtils
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat
import org.apache.spark.SparkContext
-import org.apache.spark.sql.execution.command.{ColumnProperty, Field}
+import org.apache.spark.sql.execution.command.{ColumnProperty, Field, PartitionerField}
+import org.apache.spark.sql.types.StructField
import org.apache.spark.util.FileUtils
import org.apache.carbondata.common.logging.LogServiceFactory
@@ -144,6 +145,40 @@ object CommonUtil {
isValid
}
+ /**
+ * 1. If partitioned by clause exists, then partition_type should be defined
+ * 2. If partition_type is Hash, then number_of_partitions should be defined
+ * 3. If partition_type is List, then list_info should be defined
+ * 4. If partition_type is Range, then range_info should be defined
+ * 5. Only support single level partition for now
+ * @param tableProperties
+ * @param partitionerFields
+ * @return partition clause and definition in tblproperties are valid or not
+ */
+ def validatePartitionColumns(tableProperties: Map[String, String],
+ partitionerFields: Seq[PartitionerField]): Boolean = {
+ var isValid: Boolean = true
+ val partitionType = tableProperties.get(CarbonCommonConstants.PARTITION_TYPE)
+ val numPartitions = tableProperties.get(CarbonCommonConstants.NUM_PARTITIONS)
+ val rangeInfo = tableProperties.get(CarbonCommonConstants.RANGE_INFO)
+ val listInfo = tableProperties.get(CarbonCommonConstants.LIST_INFO)
+
+ if (partitionType.isEmpty) {
+ isValid = false
+ } else {
+ partitionType.get.toUpperCase() match {
+ case "HASH" => if (!numPartitions.isDefined) isValid = false
+ case "LIST" => if (!listInfo.isDefined) isValid = false
+ case "RANGE" => if (!rangeInfo.isDefined) isValid = false
+ case "RANGE_INTERVAL" => isValid = false
+ case _ => isValid = false
+ }
+ // only support one partition column for now
+ if (partitionerFields.length > 1) isValid = false
+ }
+ isValid
+ }
+
def validateFields(key: String, fields: Seq[Field]): Boolean = {
var isValid: Boolean = false
fields.foreach { field =>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dae342bd/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index 0df3b31..2e9e61d 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -21,7 +21,7 @@ import java.util.regex.{Matcher, Pattern}
import scala.collection.JavaConverters._
import scala.collection.mutable
-import scala.collection.mutable.{LinkedHashSet, Map}
+import scala.collection.mutable.{ArrayBuffer, LinkedHashSet, ListBuffer, Map}
import scala.language.implicitConversions
import scala.util.matching.Regex
@@ -33,10 +33,13 @@ import org.apache.spark.sql.execution.command._
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.metadata.datatype.DataType
+import org.apache.carbondata.core.metadata.schema.PartitionInfo
+import org.apache.carbondata.core.metadata.schema.partition.PartitionType
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema
import org.apache.carbondata.core.util.{CarbonUtil, DataTypeUtil}
import org.apache.carbondata.processing.constants.LoggerAction
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
-import org.apache.carbondata.spark.util.CommonUtil
+import org.apache.carbondata.spark.util.{CommonUtil, DataTypeConverterUtil}
/**
* TODO remove the duplicate code and add the common methods to common class.
@@ -258,6 +261,8 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
// get no inverted index columns from table properties.
val noInvertedIdxCols = extractNoInvertedIndexColumns(fields, tableProperties)
+ // get partitionInfo
+ val partitionInfo = getPartitionInfo(partitionCols, tableProperties)
// validate the tableBlockSize from table properties
CommonUtil.validateTableBlockSize(tableProperties)
@@ -275,7 +280,8 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
Option(noInvertedIdxCols),
groupCols,
Some(colProps),
- bucketFields: Option[BucketFields])
+ bucketFields: Option[BucketFields],
+ partitionInfo)
}
/**
@@ -347,43 +353,69 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
}
/**
- * For getting the partitioner Object
- *
* @param partitionCols
* @param tableProperties
- * @return
*/
- protected def getPartitionerObject(partitionCols: Seq[PartitionerField],
- tableProperties: Map[String, String]):
- Option[Partitioner] = {
-
- // by default setting partition class empty.
- // later in table schema it is setting to default value.
- var partitionClass: String = ""
- var partitionCount: Int = 1
- var partitionColNames: Array[String] = Array[String]()
- if (tableProperties.get(CarbonCommonConstants.PARTITIONCLASS).isDefined) {
- partitionClass = tableProperties.get(CarbonCommonConstants.PARTITIONCLASS).get
- }
-
- if (tableProperties.get(CarbonCommonConstants.PARTITIONCOUNT).isDefined) {
- try {
- partitionCount = tableProperties.get(CarbonCommonConstants.PARTITIONCOUNT).get.toInt
- } catch {
- case e: Exception => // no need to do anything.
+ protected def getPartitionInfo(partitionCols: Seq[PartitionerField],
+ tableProperties: Map[String, String]): Option[PartitionInfo] = {
+ if (partitionCols.isEmpty) {
+ None
+ } else {
+ var partitionType: String = ""
+ var numPartitions = 0
+ var rangeInfo = List[String]()
+ var listInfo = ListBuffer[List[String]]()
+ var templist = ListBuffer[String]()
+ if (tableProperties.get(CarbonCommonConstants.PARTITION_TYPE).isDefined) {
+ partitionType = tableProperties.get(CarbonCommonConstants.PARTITION_TYPE).get
}
- }
-
- partitionCols.foreach(col =>
- partitionColNames :+= col.partitionColumn
- )
+ if (tableProperties.get(CarbonCommonConstants.NUM_PARTITIONS).isDefined) {
+ numPartitions = tableProperties.get(CarbonCommonConstants.NUM_PARTITIONS).get
+ .toInt
+ }
+ if (tableProperties.get(CarbonCommonConstants.RANGE_INFO).isDefined) {
+ rangeInfo = tableProperties.get(CarbonCommonConstants.RANGE_INFO).get.split(",")
+ .map(_.trim()).toList
+ }
+ if (tableProperties.get(CarbonCommonConstants.LIST_INFO).isDefined) {
+ val arr = tableProperties.get(CarbonCommonConstants.LIST_INFO).get.split(",")
+ .map(_.trim())
+ val iter = arr.iterator
+ while (iter.hasNext) {
+ val value = iter.next()
+ if (value.startsWith("(")) {
+ templist += value.replace("(", "").trim()
+ } else if (value.endsWith(")")) {
+ templist += value.replace(")", "").trim()
+ listInfo += templist.toList
+ templist.clear()
+ } else {
+ templist += value
+ listInfo += templist.toList
+ templist.clear()
+ }
+ }
+ }
+ val cols : ArrayBuffer[ColumnSchema] = new ArrayBuffer[ColumnSchema]()
+ partitionCols.foreach(partition_col => {
+ val columnSchema = new ColumnSchema
+ columnSchema.setDataType(DataTypeConverterUtil.
+ convertToCarbonType(partition_col.dataType.get))
+ columnSchema.setColumnName(partition_col.partitionColumn)
+ cols += columnSchema
+ })
- // this means user has given partition cols list
- if (!partitionColNames.isEmpty) {
- return Option(Partitioner(partitionClass, partitionColNames, partitionCount, null))
+ var partitionInfo : PartitionInfo = null
+ partitionType.toUpperCase() match {
+ case "HASH" => partitionInfo = new PartitionInfo(cols.asJava, PartitionType.HASH)
+ partitionInfo.setNumPartitions(numPartitions)
+ case "RANGE" => partitionInfo = new PartitionInfo(cols.asJava, PartitionType.RANGE)
+ partitionInfo.setRangeInfo(rangeInfo.asJava)
+ case "LIST" => partitionInfo = new PartitionInfo(cols.asJava, PartitionType.LIST)
+ partitionInfo.setListInfo(listInfo.map(_.asJava).toList.asJava)
+ }
+ Some(partitionInfo)
}
- // if partition cols are not given then no need to do partition.
- None
}
protected def extractColumnProperties(fields: Seq[Field], tableProperties: Map[String, String]):
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dae342bd/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index 187512d..6f50ecc 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -32,7 +32,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.metadata.CarbonTableIdentifier
import org.apache.carbondata.core.metadata.datatype.DataType
import org.apache.carbondata.core.metadata.encoder.Encoding
-import org.apache.carbondata.core.metadata.schema.{BucketingInfo, SchemaEvolution, SchemaEvolutionEntry}
+import org.apache.carbondata.core.metadata.schema._
import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, TableInfo, TableSchema}
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema
import org.apache.carbondata.core.service.CarbonCommonFactory
@@ -60,7 +60,8 @@ case class TableModel(
columnGroups: Seq[String],
colProps: Option[util.Map[String,
util.List[ColumnProperty]]] = None,
- bucketFields: Option[BucketFields])
+ bucketFields: Option[BucketFields],
+ partitionInfo: Option[PartitionInfo])
case class Field(column: String, var dataType: Option[String], name: Option[String],
children: Option[List[Field]], parent: String = null,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dae342bd/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
index 068f852..3aa2d4e 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
@@ -484,6 +484,8 @@ object TableCreator {
// get no inverted index columns from table properties.
val noInvertedIdxCols = extractNoInvertedIndexColumns(fields, tableProperties)
+ val partitionInfo = None
+
// validate the tableBlockSize from table properties
CommonUtil.validateTableBlockSize(tableProperties)
@@ -500,7 +502,8 @@ object TableCreator {
Option(noInvertedIdxCols),
groupCols,
Some(colProps),
- bucketFields: Option[BucketFields])
+ bucketFields: Option[BucketFields],
+ partitionInfo)
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dae342bd/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
index dc7e608..eb134d8 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
@@ -25,7 +25,8 @@ import org.apache.spark.sql.catalyst.parser.SqlBaseParser.{CreateTableContext,
TablePropertyListContext}
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.execution.SparkSqlAstBuilder
-import org.apache.spark.sql.execution.command.{BucketFields, CreateTable, Field, TableModel}
+import org.apache.spark.sql.execution.command.{BucketFields, CreateTable, Field,
+PartitionerField, TableModel}
import org.apache.spark.sql.internal.{SQLConf, VariableSubstitution}
import org.apache.carbondata.spark.CarbonOption
@@ -96,7 +97,9 @@ class CarbonSqlAstBuilder(conf: SQLConf) extends SparkSqlAstBuilder(conf) {
if (ctx.bucketSpec != null) {
operationNotAllowed("CREATE TABLE ... CLUSTERED BY", ctx)
}
- val partitionCols = Option(ctx.partitionColumns).toSeq.flatMap(visitColTypeList)
+ val partitionerFields = Option(ctx.partitionColumns).toSeq.flatMap(visitColTypeList)
+ .map( structField =>
+ PartitionerField(structField.name, Some(structField.dataType.toString), null))
val cols = Option(ctx.columns).toSeq.flatMap(visitColTypeList)
val properties = Option(ctx.tablePropertyList).map(visitPropertyKeyValues)
.getOrElse(Map.empty)
@@ -111,21 +114,23 @@ class CarbonSqlAstBuilder(conf: SQLConf) extends SparkSqlAstBuilder(conf) {
duplicateColumns.mkString("[", ",", "]"), ctx)
}
- // For Hive tables, partition columns must not be part of the schema
- val badPartCols = partitionCols.map(_.name).toSet.intersect(colNames.toSet)
- if (badPartCols.nonEmpty) {
- operationNotAllowed(s"Partition columns may not be specified in the schema: " +
- badPartCols.map("\"" + _ + "\"").mkString("[", ",", "]"), ctx)
- }
-
- // Note: Hive requires partition columns to be distinct from the schema, so we need
- // to include the partition columns here explicitly
- val schema = cols ++ partitionCols
+ val tableProperties = mutable.Map[String, String]()
+ properties.foreach{property => tableProperties.put(property._1, property._2.toLowerCase)}
- val fields = parser.getFields(schema)
+ // validate partition clause
+ if (partitionerFields.nonEmpty) {
+ if (!CommonUtil.validatePartitionColumns(tableProperties, partitionerFields)) {
+ throw new MalformedCarbonCommandException("Invalid partition definition")
+ }
+ // partition columns must be part of the schema
+ val badPartCols = partitionerFields.map(_.partitionColumn).toSet.intersect(colNames.toSet)
+ if (badPartCols.isEmpty) {
+ operationNotAllowed(s"Partition columns must be specified in the schema: " +
+ badPartCols.map("\"" + _ + "\"").mkString("[", ",", "]"), ctx)
+ }
+ }
- val tableProperties = mutable.Map[String, String]()
- properties.foreach(f => tableProperties.put(f._1, f._2.toLowerCase))
+ val fields = parser.getFields(cols)
val options = new CarbonOption(properties)
// validate tblProperties
@@ -136,7 +141,7 @@ class CarbonSqlAstBuilder(conf: SQLConf) extends SparkSqlAstBuilder(conf) {
convertDbNameToLowerCase(name.database),
name.table.toLowerCase,
fields,
- Seq(),
+ partitionerFields,
tableProperties,
bucketFields)
[15/19] carbondata git commit: [CARBONDATA-1015] Refactory write step
and add ColumnPage in data load This closes #852
Posted by ra...@apache.org.
[CARBONDATA-1015] Refactory write step and add ColumnPage in data load This closes #852
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/98df130a
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/98df130a
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/98df130a
Branch: refs/heads/master
Commit: 98df130aaaa83de9e4fe1daf38b94d3b1b7d33f6
Parents: dae342b
Author: jackylk <ja...@huawei.com>
Authored: Thu May 4 23:32:07 2017 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Sun May 14 20:40:13 2017 +0800
----------------------------------------------------------------------
.../core/compression/BigIntCompressor.java | 9 +-
.../core/compression/DoubleCompressor.java | 43 +-
.../core/compression/ValueCompressor.java | 3 +-
.../core/constants/CarbonCommonConstants.java | 6 -
.../chunk/store/MeasureChunkStoreFactory.java | 26 +-
.../BlockIndexerStorageForNoInvertedIndex.java | 2 +-
...ndexerStorageForNoInvertedIndexForShort.java | 3 +-
.../compression/MeasureMetaDataModel.java | 8 +-
.../compression/ReaderCompressModel.java | 8 +-
.../compression/ValueCompressionHolder.java | 26 +-
.../compression/WriterCompressModel.java | 41 +-
.../compression/decimal/CompressByteArray.java | 4 +-
.../decimal/CompressionMaxMinByte.java | 6 +-
.../decimal/CompressionMaxMinDefault.java | 6 +-
.../decimal/CompressionMaxMinInt.java | 6 +-
.../decimal/CompressionMaxMinLong.java | 6 +-
.../decimal/CompressionMaxMinShort.java | 6 +-
.../nondecimal/CompressionNonDecimalByte.java | 6 +-
.../CompressionNonDecimalDefault.java | 6 +-
.../nondecimal/CompressionNonDecimalInt.java | 6 +-
.../nondecimal/CompressionNonDecimalLong.java | 6 +-
.../CompressionNonDecimalMaxMinByte.java | 6 +-
.../CompressionNonDecimalMaxMinDefault.java | 6 +-
.../CompressionNonDecimalMaxMinInt.java | 6 +-
.../CompressionNonDecimalMaxMinLong.java | 6 +-
.../CompressionNonDecimalMaxMinShort.java | 6 +-
.../nondecimal/CompressionNonDecimalShort.java | 6 +-
.../compression/none/CompressionNoneByte.java | 6 +-
.../none/CompressionNoneDefault.java | 6 +-
.../compression/none/CompressionNoneInt.java | 6 +-
.../compression/none/CompressionNoneLong.java | 6 +-
.../compression/none/CompressionNoneShort.java | 6 +-
.../dataholder/CarbonWriteDataHolder.java | 156 +--
.../HeavyCompressedDoubleArrayDataStore.java | 57 --
.../core/datastore/page/ColumnPage.java | 42 +
.../core/datastore/page/ComplexColumnPage.java | 77 ++
.../datastore/page/FixLengthColumnPage.java | 155 +++
.../datastore/page/VarLengthColumnPage.java | 42 +
.../datastore/page/compression/Compression.java | 23 +
.../datastore/page/encoding/ColumnCodec.java | 35 +
.../datastore/page/encoding/DummyCodec.java | 37 +
.../page/statistics/PageStatistics.java | 124 +++
.../page/statistics/StatisticsCollector.java | 66 ++
.../core/metadata/ValueEncoderMeta.java | 18 +-
.../core/metadata/datatype/DataType.java | 6 +-
.../apache/carbondata/core/util/ByteUtil.java | 2 +
.../core/util/CarbonMetadataUtil.java | 44 +-
.../carbondata/core/util/CarbonProperties.java | 1 -
.../apache/carbondata/core/util/CarbonUtil.java | 2 +-
.../carbondata/core/util/CompressionFinder.java | 10 +-
.../carbondata/core/util/DataTypeUtil.java | 3 -
.../apache/carbondata/core/util/NodeHolder.java | 31 +-
.../core/util/ValueCompressionUtil.java | 190 ++--
.../core/util/CarbonMetadataUtilTest.java | 11 +-
.../core/util/ValueCompressionUtilTest.java | 114 +--
.../core/writer/CarbonFooterWriterTest.java | 2 +-
.../examples/CarbonSessionExample.scala | 1 +
format/src/main/thrift/carbondata.thrift | 8 +-
.../testsuite/emptyrow/TestEmptyRows.scala | 2 +-
.../dataload/TestLoadDataWithNoMeasure.scala | 8 +-
.../ColumnGroupDataTypesTestCase.scala | 6 +
.../spark/sql/parser/CarbonSparkSqlParser.scala | 1 -
.../merger/CompactionResultSortProcessor.java | 17 +-
.../sort/impl/ParallelReadMergeSorterImpl.java | 2 +-
...arallelReadMergeSorterWithBucketingImpl.java | 2 +-
.../sort/unsafe/UnsafeCarbonRowPage.java | 133 +--
.../newflow/sort/unsafe/UnsafeSortDataRows.java | 10 +-
.../holder/UnsafeSortTempFileChunkHolder.java | 29 +-
.../merger/UnsafeIntermediateFileMerger.java | 38 +-
.../CarbonRowDataWriterProcessorStepImpl.java | 10 +-
.../sortdata/IntermediateFileMerger.java | 37 +-
.../sortandgroupby/sortdata/SortDataRows.java | 29 +-
.../sortandgroupby/sortdata/SortParameters.java | 25 +-
.../sortdata/SortTempFileChunkHolder.java | 29 +-
.../store/CarbonFactDataHandlerColumnar.java | 981 ++++++++-----------
.../store/CarbonFactDataHandlerModel.java | 27 +-
.../store/SingleThreadFinalSortFilesMerger.java | 11 +-
.../store/colgroup/ColGroupBlockStorage.java | 29 +-
.../writer/v3/CarbonFactDataWriterImplV3.java | 4 +-
.../util/CarbonDataProcessorUtil.java | 50 +-
.../store/colgroup/ColGroupMinMaxTest.java | 2 +
81 files changed, 1729 insertions(+), 1309 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java b/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java
index 8e43684..8360a68 100644
--- a/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java
+++ b/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java
@@ -17,7 +17,7 @@
package org.apache.carbondata.core.compression;
import org.apache.carbondata.core.datastore.dataholder.CarbonWriteDataHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataType;
/**
* It compresses big int data
@@ -63,6 +63,7 @@ public class BigIntCompressor extends ValueCompressor {
*/
@Override
protected Object compressAdaptive(DataType convertedDataType, CarbonWriteDataHolder dataHolder) {
+
long[] value = dataHolder.getWritableLongValues();
return compressValue(convertedDataType, value, 0, false);
}
@@ -82,7 +83,7 @@ public class BigIntCompressor extends ValueCompressor {
protected Object compressValue(DataType convertedDataType, long[] value, long maxValue,
boolean isMinMax) {
switch (convertedDataType) {
- case DATA_BYTE:
+ case BYTE:
byte[] result = new byte[value.length];
if (isMinMax) {
for (int j = 0; j < value.length; j++) {
@@ -94,7 +95,7 @@ public class BigIntCompressor extends ValueCompressor {
}
}
return result;
- case DATA_SHORT:
+ case SHORT:
short[] shortResult = new short[value.length];
if (isMinMax) {
for (int j = 0; j < value.length; j++) {
@@ -106,7 +107,7 @@ public class BigIntCompressor extends ValueCompressor {
}
}
return shortResult;
- case DATA_INT:
+ case INT:
int[] intResult = new int[value.length];
if (isMinMax) {
for (int j = 0; j < value.length; j++) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java b/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java
index 840709d..bc2d6f1 100644
--- a/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java
+++ b/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java
@@ -19,7 +19,7 @@ package org.apache.carbondata.core.compression;
import java.math.BigDecimal;
import org.apache.carbondata.core.datastore.dataholder.CarbonWriteDataHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataType;
/**
* Double compressor
@@ -33,7 +33,7 @@ public class DoubleCompressor extends ValueCompressor {
BigDecimal max = BigDecimal.valueOf((double)maxValue);
double[] value = dataHolder.getWritableDoubleValues();
switch (convertedDataType) {
- case DATA_BYTE:
+ case BYTE:
byte[] result = new byte[value.length];
for (int j = 0; j < value.length; j++) {
BigDecimal val = BigDecimal.valueOf(value[j]);
@@ -42,7 +42,7 @@ public class DoubleCompressor extends ValueCompressor {
i++;
}
return result;
- case DATA_SHORT:
+ case SHORT:
short[] shortResult = new short[value.length];
for (int j = 0; j < value.length; j++) {
BigDecimal val = BigDecimal.valueOf(value[j]);
@@ -51,7 +51,7 @@ public class DoubleCompressor extends ValueCompressor {
i++;
}
return shortResult;
- case DATA_INT:
+ case INT:
int[] intResult = new int[value.length];
for (int j = 0; j < value.length; j++) {
BigDecimal val = BigDecimal.valueOf(value[j]);
@@ -60,7 +60,7 @@ public class DoubleCompressor extends ValueCompressor {
i++;
}
return intResult;
- case DATA_LONG:
+ case LONG:
long[] longResult = new long[value.length];
for (int j = 0; j < value.length; j++) {
BigDecimal val = BigDecimal.valueOf(value[j]);
@@ -69,7 +69,7 @@ public class DoubleCompressor extends ValueCompressor {
i++;
}
return longResult;
- case DATA_FLOAT:
+ case FLOAT:
float[] floatResult = new float[value.length];
for (int j = 0; j < value.length; j++) {
BigDecimal val = BigDecimal.valueOf(value[j]);
@@ -96,35 +96,35 @@ public class DoubleCompressor extends ValueCompressor {
int i = 0;
double[] value = dataHolder.getWritableDoubleValues();
switch (convertedDataType) {
- case DATA_BYTE:
+ case BYTE:
byte[] result = new byte[value.length];
for (int j = 0; j < value.length; j++) {
result[i] = (byte) (Math.round(Math.pow(10, decimal) * value[j]));
i++;
}
return result;
- case DATA_SHORT:
+ case SHORT:
short[] shortResult = new short[value.length];
for (int j = 0; j < value.length; j++) {
shortResult[i] = (short) (Math.round(Math.pow(10, decimal) * value[j]));
i++;
}
return shortResult;
- case DATA_INT:
+ case INT:
int[] intResult = new int[value.length];
for (int j = 0; j < value.length; j++) {
intResult[i] = (int) (Math.round(Math.pow(10, decimal) * value[j]));
i++;
}
return intResult;
- case DATA_LONG:
+ case LONG:
long[] longResult = new long[value.length];
for (int j = 0; j < value.length; j++) {
longResult[i] = Math.round(Math.pow(10, decimal) * value[j]);
i++;
}
return longResult;
- case DATA_FLOAT:
+ case FLOAT:
float[] floatResult = new float[value.length];
for (int j = 0; j < value.length; j++) {
floatResult[i] = (float) (Math.round(Math.pow(10, decimal) * value[j]));
@@ -148,35 +148,35 @@ public class DoubleCompressor extends ValueCompressor {
double[] value = dataHolder.getWritableDoubleValues();
int i = 0;
switch (convertedDataType) {
- case DATA_BYTE:
+ case BYTE:
byte[] result = new byte[value.length];
for (int j = 0; j < value.length; j++) {
result[i] = (byte) (maxValue - value[j]);
i++;
}
return result;
- case DATA_SHORT:
+ case SHORT:
short[] shortResult = new short[value.length];
for (int j = 0; j < value.length; j++) {
shortResult[i] = (short) (maxValue - value[j]);
i++;
}
return shortResult;
- case DATA_INT:
+ case INT:
int[] intResult = new int[value.length];
for (int j = 0; j < value.length; j++) {
intResult[i] = (int) (maxValue - value[j]);
i++;
}
return intResult;
- case DATA_LONG:
+ case LONG:
long[] longResult = new long[value.length];
for (int j = 0; j < value.length; j++) {
longResult[i] = (long) (maxValue - value[j]);
i++;
}
return longResult;
- case DATA_FLOAT:
+ case FLOAT:
float[] floatResult = new float[value.length];
for (int j = 0; j < value.length; j++) {
floatResult[i] = (float) (maxValue - value[j]);
@@ -198,36 +198,35 @@ public class DoubleCompressor extends ValueCompressor {
double[] value = dataHolder.getWritableDoubleValues();
int i = 0;
switch (changedDataType) {
- case DATA_BYTE:
+ case BYTE:
byte[] result = new byte[value.length];
for (int j = 0; j < value.length; j++) {
result[i] = (byte) value[j];
i++;
}
return result;
- case DATA_SHORT:
+ case SHORT:
short[] shortResult = new short[value.length];
for (int j = 0; j < value.length; j++) {
shortResult[i] = (short) value[j];
i++;
}
return shortResult;
- case DATA_INT:
+ case INT:
int[] intResult = new int[value.length];
for (int j = 0; j < value.length; j++) {
intResult[i] = (int) value[j];
i++;
}
return intResult;
- case DATA_LONG:
- case DATA_BIGINT:
+ case LONG:
long[] longResult = new long[value.length];
for (int j = 0; j < value.length; j++) {
longResult[i] = (long) value[j];
i++;
}
return longResult;
- case DATA_FLOAT:
+ case FLOAT:
float[] floatResult = new float[value.length];
for (int j = 0; j < value.length; j++) {
floatResult[i] = (float) value[j];
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java b/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java
index 230507f..16f8ac1 100644
--- a/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java
+++ b/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java
@@ -17,10 +17,9 @@
package org.apache.carbondata.core.compression;
import org.apache.carbondata.core.datastore.dataholder.CarbonWriteDataHolder;
+import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.util.CompressionFinder;
import org.apache.carbondata.core.util.ValueCompressionUtil.COMPRESSION_TYPE;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
/**
* Measure compressor
*/
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 33b0ef7..473e73b 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -647,12 +647,6 @@ public final class CarbonCommonConstants {
public static final char BIG_INT_MEASURE = 'd';
/**
- * This determines the size of array to be processed in data load steps. one
- * for dimensions , one of ignore dictionary dimensions , one for measures.
- */
- public static final int ARRAYSIZE = 3;
-
- /**
* CARBON_PREFETCH_BUFFERSIZE
*/
public static final String CARBON_PREFETCH_BUFFERSIZE = "carbon.prefetch.buffersize";
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/MeasureChunkStoreFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/MeasureChunkStoreFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/MeasureChunkStoreFactory.java
index 2a3a2a7..12bfea9 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/MeasureChunkStoreFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/MeasureChunkStoreFactory.java
@@ -30,8 +30,8 @@ import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe.UnsafeDouble
import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe.UnsafeIntMeasureChunkStore;
import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe.UnsafeLongMeasureChunkStore;
import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe.UnsafeShortMeasureChunkStore;
+import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.util.CarbonProperties;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
/**
* Factory class for getting the measure store type
@@ -67,33 +67,33 @@ public class MeasureChunkStoreFactory {
public MeasureDataChunkStore getMeasureDataChunkStore(DataType dataType, int numberOfRows) {
if (!isUnsafe) {
switch (dataType) {
- case DATA_BYTE:
+ case BYTE:
return new SafeByteMeasureChunkStore(numberOfRows);
- case DATA_SHORT:
+ case SHORT:
return new SafeShortMeasureChunkStore(numberOfRows);
- case DATA_INT:
+ case INT:
return new SafeIntMeasureChunkStore(numberOfRows);
- case DATA_LONG:
+ case LONG:
return new SafeLongMeasureChunkStore(numberOfRows);
- case DATA_BIGDECIMAL:
+ case DECIMAL:
return new SafeBigDecimalMeasureChunkStore(numberOfRows);
- case DATA_DOUBLE:
+ case DOUBLE:
default:
return new SafeDoubleMeasureChunkStore(numberOfRows);
}
} else {
switch (dataType) {
- case DATA_BYTE:
+ case BYTE:
return new UnsafeByteMeasureChunkStore(numberOfRows);
- case DATA_SHORT:
+ case SHORT:
return new UnsafeShortMeasureChunkStore(numberOfRows);
- case DATA_INT:
+ case INT:
return new UnsafeIntMeasureChunkStore(numberOfRows);
- case DATA_LONG:
+ case LONG:
return new UnsafeLongMeasureChunkStore(numberOfRows);
- case DATA_BIGDECIMAL:
+ case DECIMAL:
return new UnsafeBigDecimalMeasureChunkStore(numberOfRows);
- case DATA_DOUBLE:
+ case DOUBLE:
default:
return new UnsafeDoubleMeasureChunkStore(numberOfRows);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java
index 0ef2518..f36ee97 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java
@@ -37,7 +37,7 @@ public class BlockIndexerStorageForNoInvertedIndex implements IndexStorage<int[]
private byte[] min;
private byte[] max;
- public BlockIndexerStorageForNoInvertedIndex(byte[][] keyBlockInput, boolean isNoDictionary) {
+ public BlockIndexerStorageForNoInvertedIndex(byte[][] keyBlockInput) {
this.keyBlock = keyBlockInput;
min = keyBlock[0];
max = keyBlock[0];
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndexForShort.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndexForShort.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndexForShort.java
index 731df96..901084f 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndexForShort.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndexForShort.java
@@ -36,8 +36,7 @@ public class BlockIndexerStorageForNoInvertedIndexForShort implements IndexStora
private byte[] min;
private byte[] max;
- public BlockIndexerStorageForNoInvertedIndexForShort(byte[][] keyBlockInput,
- boolean isNoDictionary) {
+ public BlockIndexerStorageForNoInvertedIndexForShort(byte[][] keyBlockInput) {
this.keyBlock = keyBlockInput;
min = keyBlock[0];
max = keyBlock[0];
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/datastore/compression/MeasureMetaDataModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/MeasureMetaDataModel.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/MeasureMetaDataModel.java
index 5d35b0d..7a39f7c 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/MeasureMetaDataModel.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/MeasureMetaDataModel.java
@@ -17,6 +17,8 @@
package org.apache.carbondata.core.datastore.compression;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+
public class MeasureMetaDataModel {
/**
* maxValue
@@ -46,7 +48,7 @@ public class MeasureMetaDataModel {
/**
* type
*/
- private char[] type;
+ private DataType[] type;
/**
* dataTypeSelected
@@ -54,7 +56,7 @@ public class MeasureMetaDataModel {
private byte[] dataTypeSelected;
public MeasureMetaDataModel(Object[] minValue, Object[] maxValue, int[] mantissa,
- int measureCount, Object[] uniqueValue, char[] type, byte[] dataTypeSelected) {
+ int measureCount, Object[] uniqueValue, DataType[] type, byte[] dataTypeSelected) {
this.minValue = minValue;
this.maxValue = maxValue;
this.mantissa = mantissa;
@@ -112,7 +114,7 @@ public class MeasureMetaDataModel {
/**
* @return the type
*/
- public char[] getType() {
+ public DataType[] getType() {
return type;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/datastore/compression/ReaderCompressModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/ReaderCompressModel.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/ReaderCompressModel.java
index 101a24b..60687d1 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/ReaderCompressModel.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/ReaderCompressModel.java
@@ -18,13 +18,13 @@
package org.apache.carbondata.core.datastore.compression;
import org.apache.carbondata.core.metadata.ValueEncoderMeta;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
+import org.apache.carbondata.core.metadata.datatype.DataType;
// Used in read path for decompression preparation
public class ReaderCompressModel {
private ValueEncoderMeta valueEncoderMeta;
- private ValueCompressionUtil.DataType convertedDataType;
+ private DataType convertedDataType;
private ValueCompressionHolder valueHolder;
@@ -32,11 +32,11 @@ public class ReaderCompressModel {
this.valueEncoderMeta = valueEncoderMeta;
}
- public ValueCompressionUtil.DataType getConvertedDataType() {
+ public DataType getConvertedDataType() {
return convertedDataType;
}
- public void setConvertedDataType(ValueCompressionUtil.DataType convertedDataType) {
+ public void setConvertedDataType(DataType convertedDataType) {
this.convertedDataType = convertedDataType;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/datastore/compression/ValueCompressionHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/ValueCompressionHolder.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/ValueCompressionHolder.java
index 9b3a18a..614eb32 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/ValueCompressionHolder.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/ValueCompressionHolder.java
@@ -19,7 +19,7 @@ package org.apache.carbondata.core.datastore.compression;
import java.math.BigDecimal;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataType;
/**
* ValueCompressionHolder is the base class for handling
@@ -40,24 +40,23 @@ public abstract class ValueCompressionHolder<T> {
protected void unCompress(Compressor compressor, DataType dataType, byte[] data, int offset,
int length, int numberOfRows, Object maxValueObject, int decimalPlaces) {
switch (dataType) {
- case DATA_BYTE:
+ case BYTE:
setValue((T) compressor.unCompressByte(data, offset, length), numberOfRows, maxValueObject,
decimalPlaces);
break;
- case DATA_SHORT:
+ case SHORT:
setValue((T) compressor.unCompressShort(data, offset, length), numberOfRows, maxValueObject,
decimalPlaces);
break;
- case DATA_INT:
+ case INT:
setValue((T) compressor.unCompressInt(data, offset, length), numberOfRows, maxValueObject,
decimalPlaces);
break;
- case DATA_LONG:
- case DATA_BIGINT:
+ case LONG:
setValue((T) compressor.unCompressLong(data, offset, length), numberOfRows, maxValueObject,
decimalPlaces);
break;
- case DATA_FLOAT:
+ case FLOAT:
setValue((T) compressor.unCompressFloat(data, offset, length), numberOfRows, maxValueObject,
decimalPlaces);
break;
@@ -75,18 +74,17 @@ public abstract class ValueCompressionHolder<T> {
*/
public byte[] compress(Compressor compressor, DataType dataType, Object data) {
switch (dataType) {
- case DATA_BYTE:
+ case BYTE:
return compressor.compressByte((byte[]) data);
- case DATA_SHORT:
+ case SHORT:
return compressor.compressShort((short[]) data);
- case DATA_INT:
+ case INT:
return compressor.compressInt((int[]) data);
- case DATA_LONG:
- case DATA_BIGINT:
+ case LONG:
return compressor.compressLong((long[]) data);
- case DATA_FLOAT:
+ case FLOAT:
return compressor.compressFloat((float[]) data);
- case DATA_DOUBLE:
+ case DOUBLE:
default:
return compressor.compressDouble((double[]) data);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/datastore/compression/WriterCompressModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/WriterCompressModel.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/WriterCompressModel.java
index 0430c8c..a2bf47a 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/WriterCompressModel.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/WriterCompressModel.java
@@ -17,19 +17,24 @@
package org.apache.carbondata.core.datastore.compression;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.util.CompressionFinder;
import org.apache.carbondata.core.util.ValueCompressionUtil;
+import static org.apache.carbondata.core.metadata.datatype.DataType.INT;
+import static org.apache.carbondata.core.metadata.datatype.DataType.SHORT;
+
public class WriterCompressModel {
/**
* DataType[] variable.
*/
- private ValueCompressionUtil.DataType[] convertedDataType;
+ private DataType[] convertedDataType;
/**
* DataType[] variable.
*/
- private ValueCompressionUtil.DataType[] actualDataType;
+ private DataType[] actualDataType;
/**
* maxValue
@@ -52,7 +57,7 @@ public class WriterCompressModel {
/**
* aggType
*/
- private char[] type;
+ private DataType[] type;
/**
* dataTypeSelected
@@ -68,28 +73,28 @@ public class WriterCompressModel {
/**
* @return the convertedDataType
*/
- public ValueCompressionUtil.DataType[] getConvertedDataType() {
+ public DataType[] getConvertedDataType() {
return convertedDataType;
}
/**
* @param convertedDataType the convertedDataType to set
*/
- public void setConvertedDataType(ValueCompressionUtil.DataType[] convertedDataType) {
+ public void setConvertedDataType(DataType[] convertedDataType) {
this.convertedDataType = convertedDataType;
}
/**
* @return the actualDataType
*/
- public ValueCompressionUtil.DataType[] getActualDataType() {
+ public DataType[] getActualDataType() {
return actualDataType;
}
/**
* @param actualDataType
*/
- public void setActualDataType(ValueCompressionUtil.DataType[] actualDataType) {
+ public void setActualDataType(DataType[] actualDataType) {
this.actualDataType = actualDataType;
}
@@ -159,13 +164,33 @@ public class WriterCompressModel {
* @return the aggType
*/
public char[] getType() {
+ char[] ret = new char[type.length];
+ for (int i = 0; i < ret.length; i++) {
+ switch (type[i]) {
+ case SHORT:
+ case INT:
+ case LONG:
+ ret[i] = CarbonCommonConstants.BIG_INT_MEASURE;
+ break;
+ case DOUBLE:
+ ret[i] = CarbonCommonConstants.DOUBLE_MEASURE;
+ break;
+ case DECIMAL:
+ ret[i] = CarbonCommonConstants.BIG_DECIMAL_MEASURE;
+ break;
+ }
+ }
+ return ret;
+ }
+
+ public DataType[] getDataType() {
return type;
}
/**
* @param type the type to set
*/
- public void setType(char[] type) {
+ public void setType(DataType[] type) {
this.type = type;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressByteArray.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressByteArray.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressByteArray.java
index e517e41..7a36e66 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressByteArray.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressByteArray.java
@@ -29,7 +29,7 @@ import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.datastore.compression.Compressor;
import org.apache.carbondata.core.datastore.compression.CompressorFactory;
import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataType;
public class CompressByteArray extends ValueCompressionHolder<byte[]> {
@@ -101,7 +101,7 @@ public class CompressByteArray extends ValueCompressionHolder<byte[]> {
@Override
public void setValue(byte[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
- .getMeasureDataChunkStore(DataType.DATA_BIGDECIMAL, numberOfRows);
+ .getMeasureDataChunkStore(DataType.DECIMAL, numberOfRows);
this.measureChunkStore.putData(data);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinByte.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinByte.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinByte.java
index ed50cd3..45e3b47 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinByte.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinByte.java
@@ -26,7 +26,7 @@ import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.datastore.compression.Compressor;
import org.apache.carbondata.core.datastore.compression.CompressorFactory;
import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataType;
public class CompressionMaxMinByte extends ValueCompressionHolder<byte[]> {
@@ -68,7 +68,7 @@ public class CompressionMaxMinByte extends ValueCompressionHolder<byte[]> {
}
@Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_BYTE, value);
+ compressedValue = super.compress(compressor, DataType.BYTE, value);
}
@Override public void uncompress(DataType dataType, byte[] compressedData, int offset, int length,
@@ -103,7 +103,7 @@ public class CompressionMaxMinByte extends ValueCompressionHolder<byte[]> {
@Override
public void setValue(byte[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
- .getMeasureDataChunkStore(DataType.DATA_BYTE, numberOfRows);
+ .getMeasureDataChunkStore(DataType.BYTE, numberOfRows);
this.measureChunkStore.putData(data);
if (maxValueObject instanceof Long) {
this.maxValue = (long) maxValueObject;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinDefault.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinDefault.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinDefault.java
index 6550c46..6bd1947 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinDefault.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinDefault.java
@@ -27,8 +27,8 @@ import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.datastore.compression.Compressor;
import org.apache.carbondata.core.datastore.compression.CompressorFactory;
import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
public class CompressionMaxMinDefault extends ValueCompressionHolder<double[]> {
@@ -70,7 +70,7 @@ public class CompressionMaxMinDefault extends ValueCompressionHolder<double[]> {
}
@Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_DOUBLE, value);
+ compressedValue = super.compress(compressor, DataType.DOUBLE, value);
}
@Override public void uncompress(DataType dataType, byte[] compressedData, int offset, int length,
@@ -106,7 +106,7 @@ public class CompressionMaxMinDefault extends ValueCompressionHolder<double[]> {
@Override
public void setValue(double[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
- .getMeasureDataChunkStore(DataType.DATA_DOUBLE, numberOfRows);
+ .getMeasureDataChunkStore(DataType.DOUBLE, numberOfRows);
this.measureChunkStore.putData(data);
if (maxValueObject instanceof Long) {
this.maxValue = (long) maxValueObject;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinInt.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinInt.java
index 512cf2c..60ddfea 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinInt.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinInt.java
@@ -27,8 +27,8 @@ import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.datastore.compression.Compressor;
import org.apache.carbondata.core.datastore.compression.CompressorFactory;
import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
public class CompressionMaxMinInt extends ValueCompressionHolder<int[]> {
/**
@@ -72,7 +72,7 @@ public class CompressionMaxMinInt extends ValueCompressionHolder<int[]> {
}
@Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_INT, value);
+ compressedValue = super.compress(compressor, DataType.INT, value);
}
@Override public void setValueInBytes(byte[] value) {
@@ -102,7 +102,7 @@ public class CompressionMaxMinInt extends ValueCompressionHolder<int[]> {
@Override
public void setValue(int[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
this.measureChunkStore =
- MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_INT, numberOfRows);
+ MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.INT, numberOfRows);
this.measureChunkStore.putData(data);
if (maxValueObject instanceof Long) {
this.maxValue = (long) maxValueObject;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinLong.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinLong.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinLong.java
index ca91d44..159e741 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinLong.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinLong.java
@@ -27,8 +27,8 @@ import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.datastore.compression.Compressor;
import org.apache.carbondata.core.datastore.compression.CompressorFactory;
import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
public class CompressionMaxMinLong extends ValueCompressionHolder<long[]> {
/**
@@ -57,7 +57,7 @@ public class CompressionMaxMinLong extends ValueCompressionHolder<long[]> {
}
@Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_LONG, value);
+ compressedValue = super.compress(compressor, DataType.LONG, value);
}
@Override public void setValue(long[] value) {
@@ -102,7 +102,7 @@ public class CompressionMaxMinLong extends ValueCompressionHolder<long[]> {
@Override
public void setValue(long[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
- .getMeasureDataChunkStore(DataType.DATA_LONG, numberOfRows);
+ .getMeasureDataChunkStore(DataType.LONG, numberOfRows);
this.measureChunkStore.putData(data);
if (maxValueObject instanceof Long) {
this.maxValue = (long) maxValueObject;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinShort.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinShort.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinShort.java
index d44875a..1d36375 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinShort.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinShort.java
@@ -27,8 +27,8 @@ import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.datastore.compression.Compressor;
import org.apache.carbondata.core.datastore.compression.CompressorFactory;
import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
public class CompressionMaxMinShort extends ValueCompressionHolder<short[]> {
@@ -74,7 +74,7 @@ public class CompressionMaxMinShort extends ValueCompressionHolder<short[]> {
}
@Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_SHORT, value);
+ compressedValue = super.compress(compressor, DataType.SHORT, value);
}
@Override public void setValueInBytes(byte[] value) {
@@ -104,7 +104,7 @@ public class CompressionMaxMinShort extends ValueCompressionHolder<short[]> {
@Override
public void setValue(short[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
- .getMeasureDataChunkStore(DataType.DATA_SHORT, numberOfRows);
+ .getMeasureDataChunkStore(DataType.SHORT, numberOfRows);
this.measureChunkStore.putData(data);
if (maxValueObject instanceof Long) {
this.maxValue = (long) maxValueObject;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalByte.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalByte.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalByte.java
index 13f962f..1c46dea 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalByte.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalByte.java
@@ -26,7 +26,7 @@ import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.datastore.compression.Compressor;
import org.apache.carbondata.core.datastore.compression.CompressorFactory;
import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataType;
public class CompressionNonDecimalByte extends ValueCompressionHolder<byte[]> {
/**
@@ -58,7 +58,7 @@ public class CompressionNonDecimalByte extends ValueCompressionHolder<byte[]> {
}
@Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_BYTE, value);
+ compressedValue = super.compress(compressor, DataType.BYTE, value);
}
@Override public void uncompress(DataType dataType, byte[] compressedData, int offset, int length,
@@ -92,7 +92,7 @@ public class CompressionNonDecimalByte extends ValueCompressionHolder<byte[]> {
@Override
public void setValue(byte[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
- .getMeasureDataChunkStore(DataType.DATA_BYTE, numberOfRows);
+ .getMeasureDataChunkStore(DataType.BYTE, numberOfRows);
this.measureChunkStore.putData(data);
this.divisionFactory = Math.pow(10, decimalPlaces);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalDefault.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalDefault.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalDefault.java
index 15c7bb3..40312db 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalDefault.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalDefault.java
@@ -27,8 +27,8 @@ import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.datastore.compression.Compressor;
import org.apache.carbondata.core.datastore.compression.CompressorFactory;
import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
public class CompressionNonDecimalDefault extends ValueCompressionHolder<double[]> {
/**
@@ -56,7 +56,7 @@ public class CompressionNonDecimalDefault extends ValueCompressionHolder<double[
}
@Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_DOUBLE, value);
+ compressedValue = super.compress(compressor, DataType.DOUBLE, value);
}
@Override public void setValue(double[] value) {
@@ -94,7 +94,7 @@ public class CompressionNonDecimalDefault extends ValueCompressionHolder<double[
@Override
public void setValue(double[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
- .getMeasureDataChunkStore(DataType.DATA_DOUBLE, numberOfRows);
+ .getMeasureDataChunkStore(DataType.DOUBLE, numberOfRows);
this.measureChunkStore.putData(data);
this.divisionFactory = Math.pow(10, decimalPlaces);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalInt.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalInt.java
index 7e3606e..ada751e 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalInt.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalInt.java
@@ -27,8 +27,8 @@ import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.datastore.compression.Compressor;
import org.apache.carbondata.core.datastore.compression.CompressorFactory;
import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
public class CompressionNonDecimalInt extends ValueCompressionHolder<int[]> {
/**
@@ -58,7 +58,7 @@ public class CompressionNonDecimalInt extends ValueCompressionHolder<int[]> {
}
@Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_INT, value);
+ compressedValue = super.compress(compressor, DataType.INT, value);
}
@Override public void setValueInBytes(byte[] bytesArr) {
@@ -93,7 +93,7 @@ public class CompressionNonDecimalInt extends ValueCompressionHolder<int[]> {
@Override
public void setValue(int[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
this.measureChunkStore =
- MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_INT, numberOfRows);
+ MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.INT, numberOfRows);
this.measureChunkStore.putData(data);
this.divisionFactory = Math.pow(10, decimalPlaces);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalLong.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalLong.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalLong.java
index d810972..f0b2323 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalLong.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalLong.java
@@ -27,8 +27,8 @@ import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.datastore.compression.Compressor;
import org.apache.carbondata.core.datastore.compression.CompressorFactory;
import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
public class CompressionNonDecimalLong extends ValueCompressionHolder<long[]> {
/**
@@ -60,7 +60,7 @@ public class CompressionNonDecimalLong extends ValueCompressionHolder<long[]> {
}
@Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_LONG, value);
+ compressedValue = super.compress(compressor, DataType.LONG, value);
}
@Override public void uncompress(DataType dataType, byte[] compressedData, int offset, int length,
@@ -95,7 +95,7 @@ public class CompressionNonDecimalLong extends ValueCompressionHolder<long[]> {
@Override
public void setValue(long[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
- .getMeasureDataChunkStore(DataType.DATA_LONG, numberOfRows);
+ .getMeasureDataChunkStore(DataType.LONG, numberOfRows);
this.measureChunkStore.putData(data);
this.divisionFactory = Math.pow(10, decimalPlaces);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinByte.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinByte.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinByte.java
index 83e8a1a..c425706 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinByte.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinByte.java
@@ -26,7 +26,7 @@ import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.datastore.compression.Compressor;
import org.apache.carbondata.core.datastore.compression.CompressorFactory;
import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataType;
public class CompressionNonDecimalMaxMinByte extends ValueCompressionHolder<byte[]> {
/**
@@ -58,7 +58,7 @@ public class CompressionNonDecimalMaxMinByte extends ValueCompressionHolder<byte
}
@Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_BYTE, value);
+ compressedValue = super.compress(compressor, DataType.BYTE, value);
}
@Override public void setValueInBytes(byte[] value) {
@@ -97,7 +97,7 @@ public class CompressionNonDecimalMaxMinByte extends ValueCompressionHolder<byte
@Override
public void setValue(byte[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
- .getMeasureDataChunkStore(DataType.DATA_BYTE, numberOfRows);
+ .getMeasureDataChunkStore(DataType.BYTE, numberOfRows);
this.measureChunkStore.putData(data);
this.maxValue = BigDecimal.valueOf((double) maxValueObject);
this.divisionFactor = Math.pow(10, decimalPlaces);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinDefault.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinDefault.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinDefault.java
index c408d9a..521328e 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinDefault.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinDefault.java
@@ -27,8 +27,8 @@ import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.datastore.compression.Compressor;
import org.apache.carbondata.core.datastore.compression.CompressorFactory;
import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
public class CompressionNonDecimalMaxMinDefault extends ValueCompressionHolder<double[]> {
/**
@@ -62,7 +62,7 @@ public class CompressionNonDecimalMaxMinDefault extends ValueCompressionHolder<d
}
@Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_DOUBLE, value);
+ compressedValue = super.compress(compressor, DataType.DOUBLE, value);
}
@Override public void setValueInBytes(byte[] value) {
@@ -99,7 +99,7 @@ public class CompressionNonDecimalMaxMinDefault extends ValueCompressionHolder<d
@Override
public void setValue(double[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
- .getMeasureDataChunkStore(DataType.DATA_DOUBLE, numberOfRows);
+ .getMeasureDataChunkStore(DataType.DOUBLE, numberOfRows);
this.measureChunkStore.putData(data);
this.maxValue = BigDecimal.valueOf((double) maxValueObject);
this.divisionFactor = Math.pow(10, decimalPlaces);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinInt.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinInt.java
index 40b1099..efd21d5 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinInt.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinInt.java
@@ -27,8 +27,8 @@ import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.datastore.compression.Compressor;
import org.apache.carbondata.core.datastore.compression.CompressorFactory;
import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
public class CompressionNonDecimalMaxMinInt extends ValueCompressionHolder<int[]> {
/**
@@ -60,7 +60,7 @@ public class CompressionNonDecimalMaxMinInt extends ValueCompressionHolder<int[]
}
@Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_INT, value);
+ compressedValue = super.compress(compressor, DataType.INT, value);
}
@Override public void setValueInBytes(byte[] value) {
@@ -97,7 +97,7 @@ public class CompressionNonDecimalMaxMinInt extends ValueCompressionHolder<int[]
@Override
public void setValue(int[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
this.measureChunkStore =
- MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_INT, numberOfRows);
+ MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.INT, numberOfRows);
this.measureChunkStore.putData(data);
this.maxValue = BigDecimal.valueOf((double) maxValueObject);
this.divisionFactor = Math.pow(10, decimalPlaces);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinLong.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinLong.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinLong.java
index f8d8ed5..d2e2771 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinLong.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinLong.java
@@ -27,8 +27,8 @@ import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.datastore.compression.Compressor;
import org.apache.carbondata.core.datastore.compression.CompressorFactory;
import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
public class CompressionNonDecimalMaxMinLong extends ValueCompressionHolder<long[]> {
/**
@@ -68,7 +68,7 @@ public class CompressionNonDecimalMaxMinLong extends ValueCompressionHolder<long
}
@Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_LONG, value);
+ compressedValue = super.compress(compressor, DataType.LONG, value);
}
@Override public void setValueInBytes(byte[] value) {
@@ -98,7 +98,7 @@ public class CompressionNonDecimalMaxMinLong extends ValueCompressionHolder<long
@Override
public void setValue(long[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
- .getMeasureDataChunkStore(DataType.DATA_LONG, numberOfRows);
+ .getMeasureDataChunkStore(DataType.LONG, numberOfRows);
this.measureChunkStore.putData(data);
this.maxValue = BigDecimal.valueOf((double) maxValueObject);
this.divisionFactor = Math.pow(10, decimalPlaces);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinShort.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinShort.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinShort.java
index 14648ba..241344a 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinShort.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinShort.java
@@ -27,8 +27,8 @@ import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.datastore.compression.Compressor;
import org.apache.carbondata.core.datastore.compression.CompressorFactory;
import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
public class CompressionNonDecimalMaxMinShort extends ValueCompressionHolder<short[]> {
/**
@@ -66,7 +66,7 @@ public class CompressionNonDecimalMaxMinShort extends ValueCompressionHolder<sho
}
@Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_SHORT, value);
+ compressedValue = super.compress(compressor, DataType.SHORT, value);
}
@Override public void setValueInBytes(byte[] value) {
@@ -96,7 +96,7 @@ public class CompressionNonDecimalMaxMinShort extends ValueCompressionHolder<sho
@Override
public void setValue(short[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
- .getMeasureDataChunkStore(DataType.DATA_SHORT, numberOfRows);
+ .getMeasureDataChunkStore(DataType.SHORT, numberOfRows);
this.measureChunkStore.putData(data);
this.maxValue = BigDecimal.valueOf((double) maxValueObject);
this.divisionFactor = Math.pow(10, decimalPlaces);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalShort.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalShort.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalShort.java
index 8536630..4737e10 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalShort.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalShort.java
@@ -27,8 +27,8 @@ import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.datastore.compression.Compressor;
import org.apache.carbondata.core.datastore.compression.CompressorFactory;
import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
public class CompressionNonDecimalShort extends ValueCompressionHolder<short[]> {
/**
@@ -58,7 +58,7 @@ public class CompressionNonDecimalShort extends ValueCompressionHolder<short[]>
}
@Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_SHORT, value);
+ compressedValue = super.compress(compressor, DataType.SHORT, value);
}
@Override public void uncompress(DataType dataType, byte[] compressedData, int offset, int length,
@@ -93,7 +93,7 @@ public class CompressionNonDecimalShort extends ValueCompressionHolder<short[]>
@Override
public void setValue(short[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
- .getMeasureDataChunkStore(DataType.DATA_SHORT, numberOfRows);
+ .getMeasureDataChunkStore(DataType.SHORT, numberOfRows);
this.measureChunkStore.putData(data);
this.divisionFactory = Math.pow(10, decimalPlaces);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneByte.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneByte.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneByte.java
index acd73d9..13af929 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneByte.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneByte.java
@@ -26,7 +26,7 @@ import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.datastore.compression.Compressor;
import org.apache.carbondata.core.datastore.compression.CompressorFactory;
import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataType;
public class CompressionNoneByte extends ValueCompressionHolder<byte[]> {
/**
@@ -72,7 +72,7 @@ public class CompressionNoneByte extends ValueCompressionHolder<byte[]> {
}
@Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_BYTE, value);
+ compressedValue = super.compress(compressor, DataType.BYTE, value);
}
@Override public void setValueInBytes(byte[] value) {
@@ -98,7 +98,7 @@ public class CompressionNoneByte extends ValueCompressionHolder<byte[]> {
@Override
public void setValue(byte[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
- .getMeasureDataChunkStore(DataType.DATA_BYTE, numberOfRows);
+ .getMeasureDataChunkStore(DataType.BYTE, numberOfRows);
this.measureChunkStore.putData(data);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneDefault.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneDefault.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneDefault.java
index 8e02fd8..cca7927 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneDefault.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneDefault.java
@@ -27,8 +27,8 @@ import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.datastore.compression.Compressor;
import org.apache.carbondata.core.datastore.compression.CompressorFactory;
import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
public class CompressionNoneDefault extends ValueCompressionHolder<double[]> {
/**
@@ -69,7 +69,7 @@ public class CompressionNoneDefault extends ValueCompressionHolder<double[]> {
}
@Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_DOUBLE, value);
+ compressedValue = super.compress(compressor, DataType.DOUBLE, value);
}
@Override public void setValueInBytes(byte[] value) {
@@ -97,7 +97,7 @@ public class CompressionNoneDefault extends ValueCompressionHolder<double[]> {
@Override
public void setValue(double[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
- .getMeasureDataChunkStore(DataType.DATA_DOUBLE, numberOfRows);
+ .getMeasureDataChunkStore(DataType.DOUBLE, numberOfRows);
this.measureChunkStore.putData(data);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneInt.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneInt.java
index f0c0311..f5bb813 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneInt.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneInt.java
@@ -27,8 +27,8 @@ import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.datastore.compression.Compressor;
import org.apache.carbondata.core.datastore.compression.CompressorFactory;
import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
public class CompressionNoneInt extends ValueCompressionHolder<int[]> {
/**
@@ -62,7 +62,7 @@ public class CompressionNoneInt extends ValueCompressionHolder<int[]> {
}
@Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_INT, value);
+ compressedValue = super.compress(compressor, DataType.INT, value);
}
@Override
@@ -96,7 +96,7 @@ public class CompressionNoneInt extends ValueCompressionHolder<int[]> {
@Override
public void setValue(int[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
this.measureChunkStore =
- MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_INT, numberOfRows);
+ MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.INT, numberOfRows);
this.measureChunkStore.putData(data);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneLong.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneLong.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneLong.java
index ee72c8a..99f24ac 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneLong.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneLong.java
@@ -27,8 +27,8 @@ import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.datastore.compression.Compressor;
import org.apache.carbondata.core.datastore.compression.CompressorFactory;
import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
public class CompressionNoneLong extends ValueCompressionHolder<long[]> {
/**
@@ -62,7 +62,7 @@ public class CompressionNoneLong extends ValueCompressionHolder<long[]> {
}
@Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_LONG, value);
+ compressedValue = super.compress(compressor, DataType.LONG, value);
}
@Override
@@ -96,7 +96,7 @@ public class CompressionNoneLong extends ValueCompressionHolder<long[]> {
@Override
public void setValue(long[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
- .getMeasureDataChunkStore(DataType.DATA_LONG, numberOfRows);
+ .getMeasureDataChunkStore(DataType.LONG, numberOfRows);
this.measureChunkStore.putData(data);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneShort.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneShort.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneShort.java
index d4289ab..664f9e4 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneShort.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneShort.java
@@ -27,8 +27,8 @@ import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.datastore.compression.Compressor;
import org.apache.carbondata.core.datastore.compression.CompressorFactory;
import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
public class CompressionNoneShort extends ValueCompressionHolder<short[]> {
/**
@@ -71,7 +71,7 @@ public class CompressionNoneShort extends ValueCompressionHolder<short[]> {
}
@Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_SHORT, shortValue);
+ compressedValue = super.compress(compressor, DataType.SHORT, shortValue);
}
@Override public void setValueInBytes(byte[] value) {
@@ -98,7 +98,7 @@ public class CompressionNoneShort extends ValueCompressionHolder<short[]> {
@Override
public void setValue(short[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
- .getMeasureDataChunkStore(DataType.DATA_SHORT, numberOfRows);
+ .getMeasureDataChunkStore(DataType.SHORT, numberOfRows);
this.measureChunkStore.putData(data);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/datastore/dataholder/CarbonWriteDataHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/dataholder/CarbonWriteDataHolder.java b/core/src/main/java/org/apache/carbondata/core/datastore/dataholder/CarbonWriteDataHolder.java
index fb21d95..8d3cc0d 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/dataholder/CarbonWriteDataHolder.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/dataholder/CarbonWriteDataHolder.java
@@ -34,137 +34,57 @@ public class CarbonWriteDataHolder {
private byte[][] byteValues;
/**
- * byteValues for no dictionary.
- */
- private byte[][][] byteValuesForNonDictionary;
-
- /**
- * byteValues
- */
- private byte[][][] columnByteValues;
-
- /**
* size
*/
private int size;
/**
- * totalSize
+ * total size of the data in bytes added
*/
private int totalSize;
- /**
- * Method to initialise double array
- *
- * @param size
- */
- public void initialiseDoubleValues(int size) {
- if (size < 1) {
- throw new IllegalArgumentException("Invalid array size");
- }
- doubleValues = new double[size];
- }
-
public void reset() {
size = 0;
totalSize = 0;
}
/**
- * Method to initialise double array
- *
- * @param size
- */
- public void initialiseByteArrayValues(int size) {
- if (size < 1) {
- throw new IllegalArgumentException("Invalid array size");
- }
-
- byteValues = new byte[size][];
- columnByteValues = new byte[size][][];
- }
-
- /**
- * Method to initialise byte array
- *
- * @param size
+ * set long data type columnar data
+ * @param values
*/
- public void initialiseByteArrayValuesForKey(int size) {
- if (size < 1) {
- throw new IllegalArgumentException("Invalid array size");
- }
-
- byteValues = new byte[size][];
- }
-
- public void initialiseByteArrayValuesForNonDictionary(int size) {
- if (size < 1) {
- throw new IllegalArgumentException("Invalid array size");
+ public void setWritableLongPage(long[] values) {
+ if (values != null) {
+ longValues = values;
+ size += values.length;
+ totalSize += values.length;
}
-
- byteValuesForNonDictionary = new byte[size][][];
}
/**
- * Method to initialise long array
- *
- * @param size
+ * set double data type columnar data
+ * @param values
*/
- public void initialiseLongValues(int size) {
- if (size < 1) {
- throw new IllegalArgumentException("Invalid array size");
+ public void setWritableDoublePage(double[] values) {
+ if (values != null) {
+ doubleValues = values;
+ size += values.length;
+ totalSize += values.length;
}
- longValues = new long[size];
- }
-
- /**
- * set double value by index
- *
- * @param index
- * @param value
- */
- public void setWritableDoubleValueByIndex(int index, Object value) {
- doubleValues[index] = (Double) value;
- size++;
}
/**
- * set double value by index
- *
- * @param index
- * @param value
- */
- public void setWritableLongValueByIndex(int index, Object value) {
- longValues[index] = (Long) value;
- size++;
- }
-
- /**
- * set byte array value by index
- *
- * @param index
- * @param value
+ * set decimal data type columnar data
+ * @param values
*/
- public void setWritableByteArrayValueByIndex(int index, byte[] value) {
- byteValues[index] = value;
- size++;
- if (null != value) totalSize += value.length;
- }
-
- public void setWritableNonDictByteArrayValueByIndex(int index, byte[][] value) {
- byteValuesForNonDictionary[index] = value;
- size++;
- if (null != value) totalSize += value.length;
- }
-
- /**
- * set byte array value by index
- */
- public void setWritableByteArrayValueByIndex(int index, int mdKeyIndex, Object[] columnData) {
- int l = 0;
- columnByteValues[index] = new byte[columnData.length - (mdKeyIndex + 1)][];
- for (int i = mdKeyIndex + 1; i < columnData.length; i++) {
- columnByteValues[index][l++] = (byte[]) columnData[i];
+ public void setWritableDecimalPage(byte[][] values) {
+ if (values != null) {
+ byteValues = values;
+ size += values.length;
+ for (int i = 0; i < values.length; i++) {
+ if (values[i] != null) {
+ totalSize += values[i].length;
+ }
+ }
}
}
@@ -187,30 +107,14 @@ public class CarbonWriteDataHolder {
byte[] temp = new byte[totalSize];
int startIndexToCopy = 0;
for (int i = 0; i < size; i++) {
- System.arraycopy(byteValues[i], 0, temp, startIndexToCopy, byteValues[i].length);
- startIndexToCopy += byteValues[i].length;
+ if (byteValues[i] != null) {
+ System.arraycopy(byteValues[i], 0, temp, startIndexToCopy, byteValues[i].length);
+ startIndexToCopy += byteValues[i].length;
+ }
}
return temp;
}
- public byte[][] getByteArrayValues() {
- if (size < byteValues.length) {
- byte[][] temp = new byte[size][];
- System.arraycopy(byteValues, 0, temp, 0, size);
- byteValues = temp;
- }
- return byteValues;
- }
-
- public byte[][][] getNonDictByteArrayValues() {
- if (size < byteValuesForNonDictionary.length) {
- byte[][][] temp = new byte[size][][];
- System.arraycopy(byteValuesForNonDictionary, 0, temp, 0, size);
- byteValuesForNonDictionary = temp;
- }
- return byteValuesForNonDictionary;
- }
-
/**
* Get Writable Double Values
*
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/datastore/impl/data/compressed/HeavyCompressedDoubleArrayDataStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/data/compressed/HeavyCompressedDoubleArrayDataStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/data/compressed/HeavyCompressedDoubleArrayDataStore.java
deleted file mode 100644
index d3d67fd..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/data/compressed/HeavyCompressedDoubleArrayDataStore.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.datastore.impl.data.compressed;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.datastore.compression.WriterCompressModel;
-import org.apache.carbondata.core.datastore.dataholder.CarbonWriteDataHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-
-public class HeavyCompressedDoubleArrayDataStore {
-
- // this method first invokes encoding routine to encode the data chunk,
- // followed by invoking compression routine for preparing the data chunk for writing.
- public static byte[][] encodeMeasureDataArray(
- WriterCompressModel compressionModel,
- CarbonWriteDataHolder[] dataHolder) {
- char[] type = compressionModel.getType();
- ValueCompressionHolder[] values =
- new ValueCompressionHolder[compressionModel.getValueCompressionHolder().length];
- byte[][] returnValue = new byte[values.length][];
- for (int i = 0; i < compressionModel.getValueCompressionHolder().length; i++) {
- values[i] = compressionModel.getValueCompressionHolder()[i];
- if (type[i] != CarbonCommonConstants.BYTE_VALUE_MEASURE
- && type[i] != CarbonCommonConstants.BIG_DECIMAL_MEASURE) {
- // first perform encoding of the data chunk
- values[i].setValue(
- ValueCompressionUtil.getValueCompressor(compressionModel.getCompressionFinders()[i])
- .getCompressedValues(compressionModel.getCompressionFinders()[i], dataHolder[i],
- compressionModel.getMaxValue()[i],
- compressionModel.getMantissa()[i]));
- } else {
- values[i].setValue(dataHolder[i].getWritableByteArrayValues());
- }
- values[i].compress();
- returnValue[i] = values[i].getCompressedData();
- }
-
- return returnValue;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
new file mode 100644
index 0000000..25a813c
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.datastore.page;
+
+import org.apache.carbondata.core.datastore.page.statistics.PageStatistics;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+
+public class ColumnPage {
+
+ protected final DataType dataType;
+ protected final int pageSize;
+ protected PageStatistics stats;
+
+ protected ColumnPage(DataType dataType, int pageSize) {
+ this.dataType = dataType;
+ this.pageSize = pageSize;
+ this.stats = new PageStatistics(dataType);
+ }
+
+ protected void updateStatistics(Object value) {
+ stats.update(value);
+ }
+
+ public PageStatistics getStatistics() {
+ return stats;
+ }
+}
[12/19] carbondata git commit: [CARBONDATA-1015] Refactory write step
and add ColumnPage in data load This closes #852
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
index 6d81d59..9a7450a 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
@@ -25,6 +25,7 @@ import java.math.BigDecimal;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.BitSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
@@ -48,9 +49,12 @@ import org.apache.carbondata.core.datastore.columnar.BlockIndexerStorageForNoInv
import org.apache.carbondata.core.datastore.columnar.BlockIndexerStorageForShort;
import org.apache.carbondata.core.datastore.columnar.ColumnGroupModel;
import org.apache.carbondata.core.datastore.columnar.IndexStorage;
+import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
import org.apache.carbondata.core.datastore.compression.WriterCompressModel;
import org.apache.carbondata.core.datastore.dataholder.CarbonWriteDataHolder;
-import org.apache.carbondata.core.datastore.impl.data.compressed.HeavyCompressedDoubleArrayDataStore;
+import org.apache.carbondata.core.datastore.page.ComplexColumnPage;
+import org.apache.carbondata.core.datastore.page.FixLengthColumnPage;
+import org.apache.carbondata.core.datastore.page.VarLengthColumnPage;
import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.keygenerator.KeyGenerator;
import org.apache.carbondata.core.keygenerator.columnar.ColumnarSplitter;
@@ -58,6 +62,7 @@ import org.apache.carbondata.core.keygenerator.columnar.impl.MultiDimKeyVarLengt
import org.apache.carbondata.core.keygenerator.factory.KeyGeneratorFactory;
import org.apache.carbondata.core.metadata.CarbonMetadata;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
+import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.util.CarbonProperties;
@@ -67,10 +72,6 @@ import org.apache.carbondata.core.util.NodeHolder;
import org.apache.carbondata.core.util.ValueCompressionUtil;
import org.apache.carbondata.processing.datatypes.GenericDataType;
import org.apache.carbondata.processing.store.colgroup.ColGroupBlockStorage;
-import org.apache.carbondata.processing.store.colgroup.ColGroupDataHolder;
-import org.apache.carbondata.processing.store.colgroup.ColGroupMinMax;
-import org.apache.carbondata.processing.store.colgroup.ColumnDataHolder;
-import org.apache.carbondata.processing.store.colgroup.DataHolder;
import org.apache.carbondata.processing.store.file.FileManager;
import org.apache.carbondata.processing.store.file.IFileManagerComposite;
import org.apache.carbondata.processing.store.writer.CarbonDataWriterVo;
@@ -116,7 +117,8 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
*/
private int mdKeyIndex;
/**
- * blocklet size
+ * blocklet size (for V1 and V2) or page size (for V3). A Producer thread will start to process
+ * once this size of input is reached
*/
private int blockletSize;
/**
@@ -140,14 +142,6 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
*/
private int tableBlockSize;
/**
- * otherMeasureIndex
- */
- private int[] otherMeasureIndex;
- /**
- * customMeasureIndex
- */
- private int[] customMeasureIndex;
- /**
* dimLens
*/
private int[] dimLens;
@@ -161,18 +155,8 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
private CarbonKeyBlockHolder[] keyBlockHolder;
private boolean[] aggKeyBlock;
private boolean[] isNoDictionary;
- private boolean isAggKeyBlock;
private long processedDataCount;
- /**
- * thread pool size to be used for block sort
- */
- private int thread_pool_size;
private KeyGenerator[] complexKeyGenerator;
- /**
- * isDataWritingRequest
- */
- // private boolean isDataWritingRequest;
-
private ExecutorService producerExecutorService;
private List<Future<Void>> producerExecutorServiceTaskList;
private ExecutorService consumerExecutorService;
@@ -181,7 +165,7 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
private int noDictionaryCount;
private ColumnGroupModel colGrpModel;
private int[] primitiveDimLens;
- private char[] type;
+ private DataType[] type;
private int[] completeDimLens;
private boolean[] isUseInvertedIndex;
/**
@@ -201,10 +185,6 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
*/
private BlockletDataHolder blockletDataHolder;
/**
- * a private class which will take each blocklet in order and write to a file
- */
- private Consumer consumer;
- /**
* number of cores configured
*/
private int numberOfCores;
@@ -227,20 +207,15 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
private int complexColCount;
/**
- * no of column blocks
- */
- private int columnStoreCount;
-
- /**
* column schema present in the table
*/
private List<ColumnSchema> wrapperColumnSchemaList;
/**
* boolean to check whether dimension
- * is of dictionary type or no dictionary time
+ * is of dictionary type or no dictionary type
*/
- private boolean[] dimensionType;
+ private boolean[] isDictDimension;
/**
* colCardinality for the merge case.
@@ -260,8 +235,6 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
private long schemaUpdatedTimeStamp;
- private String segmentId;
-
private int taskExtension;
/**
@@ -277,19 +250,15 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
this.dimensionCount = carbonFactDataHandlerModel.getDimensionCount();
this.complexIndexMap = carbonFactDataHandlerModel.getComplexIndexMap();
this.primitiveDimLens = carbonFactDataHandlerModel.getPrimitiveDimLens();
- this.isAggKeyBlock = Boolean.parseBoolean(CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.AGGREAGATE_COLUMNAR_KEY_BLOCK,
- CarbonCommonConstants.AGGREAGATE_COLUMNAR_KEY_BLOCK_DEFAULTVALUE));
this.carbonDataDirectoryPath = carbonFactDataHandlerModel.getCarbonDataDirectoryPath();
- this.complexColCount = getComplexColsCount();
- this.columnStoreCount =
- this.colGrpModel.getNoOfColumnStore() + noDictionaryCount + complexColCount;
+ this.complexColCount = getExpandedComplexColsCount();
- this.aggKeyBlock = new boolean[columnStoreCount];
- this.isNoDictionary = new boolean[columnStoreCount];
+ int numDimColumns = colGrpModel.getNoOfColumnStore() + noDictionaryCount + complexColCount;
+ this.aggKeyBlock = new boolean[numDimColumns];
+ this.isNoDictionary = new boolean[numDimColumns];
this.bucketNumber = carbonFactDataHandlerModel.getBucketId();
this.taskExtension = carbonFactDataHandlerModel.getTaskExtension();
- this.isUseInvertedIndex = new boolean[columnStoreCount];
+ this.isUseInvertedIndex = new boolean[numDimColumns];
if (null != carbonFactDataHandlerModel.getIsUseInvertedIndex()) {
for (int i = 0; i < isUseInvertedIndex.length; i++) {
if (i < carbonFactDataHandlerModel.getIsUseInvertedIndex().length) {
@@ -305,6 +274,9 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
this.isNoDictionary[noDictStartIndex + i] = true;
}
+ boolean isAggKeyBlock = Boolean.parseBoolean(CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.AGGREAGATE_COLUMNAR_KEY_BLOCK,
+ CarbonCommonConstants.AGGREAGATE_COLUMNAR_KEY_BLOCK_DEFAULTVALUE));
if (isAggKeyBlock) {
int noDictionaryValue = Integer.parseInt(CarbonProperties.getInstance()
.getProperty(CarbonCommonConstants.HIGH_CARDINALITY_VALUE,
@@ -346,7 +318,7 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
this.databaseName = carbonFactDataHandlerModel.getDatabaseName();
this.tableBlockSize = carbonFactDataHandlerModel.getBlockSizeInMB();
this.tableName = carbonFactDataHandlerModel.getTableName();
- this.type = carbonFactDataHandlerModel.getAggType();
+ this.type = carbonFactDataHandlerModel.getMeasureDataType();
this.segmentProperties = carbonFactDataHandlerModel.getSegmentProperties();
this.wrapperColumnSchemaList = carbonFactDataHandlerModel.getWrapperColumnSchema();
this.colCardinality = carbonFactDataHandlerModel.getColCardinality();
@@ -360,11 +332,11 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
this.dimLens = this.segmentProperties.getDimColumnsCardinality();
this.carbonDataFileAttributes = carbonFactDataHandlerModel.getCarbonDataFileAttributes();
this.schemaUpdatedTimeStamp = carbonFactDataHandlerModel.getSchemaUpdatedTimeStamp();
- this.segmentId = carbonFactDataHandlerModel.getSegmentId();
+
//TODO need to pass carbon table identifier to metadata
CarbonTable carbonTable = CarbonMetadata.getInstance()
.getCarbonTable(databaseName + CarbonCommonConstants.UNDERSCORE + tableName);
- dimensionType =
+ isDictDimension =
CarbonUtil.identifyDimensionType(carbonTable.getDimensionByTableName(tableName));
this.compactionFlow = carbonFactDataHandlerModel.isCompactionFlow();
@@ -403,15 +375,17 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
consumerExecutorServiceTaskList = new ArrayList<>(1);
semaphore = new Semaphore(numberOfCores);
blockletDataHolder = new BlockletDataHolder();
- consumer = new Consumer(blockletDataHolder);
+
+ // Start the consumer which will take each blocklet/page in order and write to a file
+ Consumer consumer = new Consumer(blockletDataHolder);
consumerExecutorServiceTaskList.add(consumerExecutorService.submit(consumer));
}
private boolean[] arrangeUniqueBlockType(boolean[] aggKeyBlock) {
int counter = 0;
boolean[] uniqueBlock = new boolean[aggKeyBlock.length];
- for (int i = 0; i < dimensionType.length; i++) {
- if (dimensionType[i]) {
+ for (int i = 0; i < isDictDimension.length; i++) {
+ if (isDictDimension[i]) {
uniqueBlock[i] = aggKeyBlock[counter++];
} else {
uniqueBlock[i] = false;
@@ -463,8 +437,12 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
if (this.entryCount == this.blockletSize) {
try {
semaphore.acquire();
- producerExecutorServiceTaskList.add(producerExecutorService.submit(
- new Producer(blockletDataHolder, dataRows, ++writerTaskSequenceCounter, false)));
+
+ producerExecutorServiceTaskList.add(
+ producerExecutorService.submit(
+ new Producer(blockletDataHolder, dataRows, ++writerTaskSequenceCounter, false)
+ )
+ );
blockletProcessingCount.incrementAndGet();
// set the entry count to zero
processedDataCount += entryCount;
@@ -478,417 +456,268 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
}
}
- /** statics for one blocklet/page */
- class Statistics {
- /** min and max value of the measures */
- Object[] min, max;
-
- /**
- * the unique value is the non-exist value in the row,
- * and will be used as storage key for null values of measures
- */
- Object[] uniqueValue;
-
- /** decimal count of the measures */
- int[] decimal;
-
- Statistics(int measureCount) {
- max = new Object[measureCount];
- min = new Object[measureCount];
- uniqueValue = new Object[measureCount];
- decimal = new int[measureCount];
- for (int i = 0; i < measureCount; i++) {
- if (type[i] == CarbonCommonConstants.BIG_INT_MEASURE) {
- max[i] = Long.MIN_VALUE;
- min[i] = Long.MAX_VALUE;
- uniqueValue[i] = Long.MIN_VALUE;
- } else if (type[i] == CarbonCommonConstants.DOUBLE_MEASURE) {
- max[i] = Double.MIN_VALUE;
- min[i] = Double.MAX_VALUE;
- uniqueValue[i] = Double.MIN_VALUE;
- } else if (type[i] == CarbonCommonConstants.BIG_DECIMAL_MEASURE) {
- max[i] = new BigDecimal(Double.MIN_VALUE);
- min[i] = new BigDecimal(Double.MAX_VALUE);
- uniqueValue[i] = new BigDecimal(Double.MIN_VALUE);
- } else {
- max[i] = 0.0;
- min[i] = 0.0;
- uniqueValue[i] = 0.0;
- }
- decimal[i] = 0;
- }
- }
-
- /**
- * update the statistics for the input row
- */
- void update(int[] msrIndex, Object[] row, boolean compactionFlow) {
- // Update row level min max
- for (int i = 0; i < msrIndex.length; i++) {
- int count = msrIndex[i];
- if (row[count] != null) {
- if (type[count] == CarbonCommonConstants.DOUBLE_MEASURE) {
- double value = (double) row[count];
- double maxVal = (double) max[count];
- double minVal = (double) min[count];
- max[count] = (maxVal > value ? max[count] : value);
- min[count] = (minVal < value ? min[count] : value);
- int num = getDecimalCount(value);
- decimal[count] = (decimal[count] > num ? decimal[count] : num);
- uniqueValue[count] = (double) min[count] - 1;
- } else if (type[count] == CarbonCommonConstants.BIG_INT_MEASURE) {
- long value = (long) row[count];
- long maxVal = (long) max[count];
- long minVal = (long) min[count];
- max[count] = (maxVal > value ? max[count] : value);
- min[count] = (minVal < value ? min[count] : value);
- uniqueValue[count] = (long) min[count] - 1;
- } else if (type[count] == CarbonCommonConstants.BIG_DECIMAL_MEASURE) {
- byte[] buff = null;
- // in compaction flow the measure with decimal type will come as spark decimal.
- // need to convert it to byte array.
- if (compactionFlow) {
- BigDecimal bigDecimal = ((Decimal) row[count]).toJavaBigDecimal();
- buff = DataTypeUtil.bigDecimalToByte(bigDecimal);
- } else {
- buff = (byte[]) row[count];
- }
- BigDecimal value = DataTypeUtil.byteToBigDecimal(buff);
- decimal[count] = value.scale();
- BigDecimal val = (BigDecimal) min[count];
- uniqueValue[count] = (val.subtract(new BigDecimal(1.0)));
- }
- }
- }
- }
- }
-
class IndexKey {
+ private int pageSize;
byte[] currentMDKey = null;
byte[][] currentNoDictionaryKey = null;
byte[] startKey = null;
byte[] endKey = null;
byte[][] noDictStartKey = null;
byte[][] noDictEndKey = null;
+ byte[] packedNoDictStartKey = null;
+ byte[] packedNoDictEndKey = null;
+
+ IndexKey(int pageSize) {
+ this.pageSize = pageSize;
+ }
/** update all keys based on the input row */
- void update(Object[] row, boolean firstRow) {
+ void update(int rowId, Object[] row) {
currentMDKey = (byte[]) row[mdKeyIndex];
if (noDictionaryCount > 0 || complexIndexMap.size() > 0) {
currentNoDictionaryKey = (byte[][]) row[mdKeyIndex - 1];
}
- if (firstRow) {
+ if (rowId == 0) {
startKey = currentMDKey;
noDictStartKey = currentNoDictionaryKey;
}
endKey = currentMDKey;
noDictEndKey = currentNoDictionaryKey;
- }
- }
-
- /** generate the NodeHolder from the input rows */
- private NodeHolder processDataRows(List<Object[]> dataRows)
- throws CarbonDataWriterException {
- if (dataRows.size() == 0) {
- return new NodeHolder();
- }
- // to store index of the measure columns which are null
- BitSet[] nullValueIndexBitSet = getMeasureNullValueIndexBitSet(measureCount);
- // statistics for one blocklet/page
- Statistics stats = new Statistics(measureCount);
- IndexKey keys = new IndexKey();
-
- // initialize measureHolder, mdKeyHolder and noDictionaryHolder, these three Holders
- // are the input for final encoding
- CarbonWriteDataHolder[] measureHolder = initialiseDataHolder(dataRows.size());
- CarbonWriteDataHolder mdKeyHolder = initialiseKeyBlockHolder(dataRows.size());
- CarbonWriteDataHolder noDictionaryHolder = null;
- if ((noDictionaryCount + complexColCount) > 0) {
- noDictionaryHolder = initialiseKeyBlockHolderForNonDictionary(dataRows.size());
- }
-
- // loop on the input rows, fill measureHolder, mdKeyHolder and noDictionaryHolder
- for (int count = 0; count < dataRows.size(); count++) {
- Object[] row = dataRows.get(count);
- keys.update(row, (count == 0));
- if (keys.currentMDKey.length > 0) {
- mdKeyHolder.setWritableByteArrayValueByIndex(count, keys.currentMDKey);
- }
- if (noDictionaryCount > 0 || complexIndexMap.size() > 0) {
- noDictionaryHolder.setWritableNonDictByteArrayValueByIndex(count,
- keys.currentNoDictionaryKey);
+ if (rowId == pageSize - 1) {
+ finalizeKeys();
}
- fillMeasureHolder(row, count, measureHolder, nullValueIndexBitSet);
- stats.update(otherMeasureIndex, row, compactionFlow);
- stats.update(customMeasureIndex, row, compactionFlow);
}
- // generate encoded byte array for 3 holders
- // for measure columns: encode and compress the measureHolder
- WriterCompressModel compressionModel =
- ValueCompressionUtil.getWriterCompressModel(
- stats.max, stats.min, stats.decimal, stats.uniqueValue, type, new byte[measureCount]);
- byte[][] encodedMeasureArray =
- HeavyCompressedDoubleArrayDataStore.encodeMeasureDataArray(
- compressionModel, measureHolder);
-
- // for mdkey and noDictionary, it is already in bytes, just get the array from holder
- byte[][] mdKeyArray = mdKeyHolder.getByteArrayValues();
- byte[][][] noDictionaryArray = null;
- if ((noDictionaryCount + complexColCount) > 0) {
- noDictionaryArray = noDictionaryHolder.getNonDictByteArrayValues();
- }
-
- // create NodeHolder using these encoded byte arrays
- NodeHolder nodeHolder =
- createNodeHolderObjectWithOutKettle(
- encodedMeasureArray, mdKeyArray, noDictionaryArray, dataRows.size(),
- keys.startKey, keys.endKey, compressionModel, keys.noDictStartKey, keys.noDictEndKey,
- nullValueIndexBitSet);
- LOGGER.info("Number Of records processed: " + dataRows.size());
- return nodeHolder;
- }
-
- private void fillMeasureHolder(Object[] row, int count, CarbonWriteDataHolder[] measureHolder,
- BitSet[] nullValueIndexBitSet) {
- for (int k = 0; k < otherMeasureIndex.length; k++) {
- if (type[otherMeasureIndex[k]] == CarbonCommonConstants.BIG_INT_MEASURE) {
- if (null == row[otherMeasureIndex[k]]) {
- nullValueIndexBitSet[otherMeasureIndex[k]].set(count);
- measureHolder[otherMeasureIndex[k]].setWritableLongValueByIndex(count, 0L);
- } else {
- measureHolder[otherMeasureIndex[k]]
- .setWritableLongValueByIndex(count, row[otherMeasureIndex[k]]);
+ /** update all keys if SORT_COLUMNS option is used when creating table */
+ private void finalizeKeys() {
+ // If SORT_COLUMNS is used, may need to update start/end keys since the they may
+ // contains dictionary columns that are not in SORT_COLUMNS, which need to be removed from
+ // start/end key
+ int numberOfDictSortColumns = segmentProperties.getNumberOfDictSortColumns();
+ if (numberOfDictSortColumns > 0) {
+ // if SORT_COLUMNS contain dictionary columns
+ int[] keySize = columnarSplitter.getBlockKeySize();
+ if (keySize.length > numberOfDictSortColumns) {
+ // if there are some dictionary columns that are not in SORT_COLUMNS, it will come to here
+ int newMdkLength = 0;
+ for (int i = 0; i < numberOfDictSortColumns; i++) {
+ newMdkLength += keySize[i];
+ }
+ byte[] newStartKeyOfSortKey = new byte[newMdkLength];
+ byte[] newEndKeyOfSortKey = new byte[newMdkLength];
+ System.arraycopy(startKey, 0, newStartKeyOfSortKey, 0, newMdkLength);
+ System.arraycopy(endKey, 0, newEndKeyOfSortKey, 0, newMdkLength);
+ startKey = newStartKeyOfSortKey;
+ endKey = newEndKeyOfSortKey;
}
} else {
- if (null == row[otherMeasureIndex[k]]) {
- nullValueIndexBitSet[otherMeasureIndex[k]].set(count);
- measureHolder[otherMeasureIndex[k]].setWritableDoubleValueByIndex(count, 0.0);
- } else {
- measureHolder[otherMeasureIndex[k]]
- .setWritableDoubleValueByIndex(count, row[otherMeasureIndex[k]]);
- }
+ startKey = new byte[0];
+ endKey = new byte[0];
}
- }
- ByteBuffer byteBuffer = null;
- byte[] measureBytes = null;
- for (int i = 0; i < customMeasureIndex.length; i++) {
- if (null == row[customMeasureIndex[i]]
- && type[customMeasureIndex[i]] == CarbonCommonConstants.BIG_DECIMAL_MEASURE) {
- measureBytes = DataTypeUtil.zeroBigDecimalBytes;
- nullValueIndexBitSet[customMeasureIndex[i]].set(count);
- } else {
- if (this.compactionFlow) {
- BigDecimal bigDecimal = ((Decimal) row[customMeasureIndex[i]]).toJavaBigDecimal();
- measureBytes = DataTypeUtil.bigDecimalToByte(bigDecimal);
- } else {
- measureBytes = (byte[]) row[customMeasureIndex[i]];
+
+ // Do the same update for noDictionary start/end Key
+ int numberOfNoDictSortColumns = segmentProperties.getNumberOfNoDictSortColumns();
+ if (numberOfNoDictSortColumns > 0) {
+ // if sort_columns contain no-dictionary columns
+ if (noDictStartKey.length > numberOfNoDictSortColumns) {
+ byte[][] newNoDictionaryStartKey = new byte[numberOfNoDictSortColumns][];
+ byte[][] newNoDictionaryEndKey = new byte[numberOfNoDictSortColumns][];
+ System.arraycopy(
+ noDictStartKey, 0, newNoDictionaryStartKey, 0, numberOfNoDictSortColumns);
+ System.arraycopy(
+ noDictEndKey, 0, newNoDictionaryEndKey, 0, numberOfNoDictSortColumns);
+ noDictStartKey = newNoDictionaryStartKey;
+ noDictEndKey = newNoDictionaryEndKey;
}
+ packedNoDictStartKey =
+ NonDictionaryUtil.packByteBufferIntoSingleByteArray(noDictStartKey);
+ packedNoDictEndKey =
+ NonDictionaryUtil.packByteBufferIntoSingleByteArray(noDictEndKey);
}
- byteBuffer = ByteBuffer.allocate(measureBytes.length +
- CarbonCommonConstants.INT_SIZE_IN_BYTE);
- byteBuffer.putInt(measureBytes.length);
- byteBuffer.put(measureBytes);
- byteBuffer.flip();
- measureBytes = byteBuffer.array();
- measureHolder[customMeasureIndex[i]].setWritableByteArrayValueByIndex(count, measureBytes);
}
}
- private NodeHolder createNodeHolderObjectWithOutKettle(byte[][] measureArray, byte[][] mdKeyArray,
- byte[][][] noDictionaryArray, int entryCountLocal, byte[] startkeyLocal, byte[] endKeyLocal,
- WriterCompressModel compressionModel, byte[][] noDictionaryStartKey,
- byte[][] noDictionaryEndKey, BitSet[] nullValueIndexBitSet)
- throws CarbonDataWriterException {
- byte[][][] noDictionaryColumnsData = null;
- List<ArrayList<byte[]>> colsAndValues = new ArrayList<ArrayList<byte[]>>();
- int complexColCount = getComplexColsCount();
-
- for (int i = 0; i < complexColCount; i++) {
- colsAndValues.add(new ArrayList<byte[]>());
+ /**
+ * Represent a page data for all columns, we store its data in columnar layout, so that
+ * all processing apply to TablePage can be done in vectorized fashion.
+ */
+ class TablePage {
+
+ // For all dimension and measure columns, we store the column data directly in the page,
+ // the length of the page is the number of rows.
+
+ // TODO: we should have separate class for key columns so that keys are stored together in
+ // one vector to make it efficient for sorting
+ VarLengthColumnPage[] dictDimensionPage;
+ VarLengthColumnPage[] noDictDimensionPage;
+ ComplexColumnPage[] complexDimensionPage;
+ FixLengthColumnPage[] measurePage;
+
+ // the num of rows in this page, it must be less than short value (65536)
+ int pageSize;
+
+ TablePage(int pageSize) {
+ this.pageSize = pageSize;
+ dictDimensionPage = new VarLengthColumnPage[dimensionCount];
+ for (int i = 0; i < dictDimensionPage.length; i++) {
+ dictDimensionPage[i] = new VarLengthColumnPage(pageSize);
+ }
+ noDictDimensionPage = new VarLengthColumnPage[noDictionaryCount];
+ for (int i = 0; i < noDictDimensionPage.length; i++) {
+ noDictDimensionPage[i] = new VarLengthColumnPage(pageSize);
+ }
+ complexDimensionPage = new ComplexColumnPage[getComplexColumnCount()];
+ for (int i = 0; i < complexDimensionPage.length; i++) {
+ // here we still do not the depth of the complex column, it will be initialized when
+ // we get the first row.
+ complexDimensionPage[i] = null;
+ }
+ measurePage = new FixLengthColumnPage[measureCount];
+ for (int i = 0; i < measurePage.length; i++) {
+ measurePage[i] = new FixLengthColumnPage(type[i], pageSize);
+ }
}
- int noOfColumn = colGrpModel.getNoOfColumnStore();
- DataHolder[] dataHolders = getDataHolders(noOfColumn, mdKeyArray.length);
- for (int i = 0; i < mdKeyArray.length; i++) {
- byte[][] splitKey = columnarSplitter.splitKey(mdKeyArray[i]);
- for (int j = 0; j < splitKey.length; j++) {
- dataHolders[j].addData(splitKey[j], i);
+ /**
+ * Add one row to the internal store, it will be converted into columnar layout
+ * @param rowId Id of the input row
+ * @param rows row object
+ */
+ void addRow(int rowId, Object[] rows) {
+
+ // convert dictionary columns
+ byte[] MDKey = (byte[]) rows[mdKeyIndex];
+ if (columnarSplitter != null) {
+ byte[][] splitKey = columnarSplitter.splitKey(MDKey);
+ for (int i = 0; i < splitKey.length; i++) {
+ dictDimensionPage[i].putByteArray(rowId, splitKey[i]);
+ }
}
- }
- if (noDictionaryCount > 0 || complexIndexMap.size() > 0) {
- noDictionaryColumnsData = new byte[noDictionaryCount][noDictionaryArray.length][];
- for (int i = 0; i < noDictionaryArray.length; i++) {
- int complexColumnIndex = primitiveDimLens.length + noDictionaryCount;
- byte[][] splitKey = noDictionaryArray[i];
-
- int complexTypeIndex = 0;
- for (int j = 0; j < splitKey.length; j++) {
- //nodictionary Columns
- if (j < noDictionaryCount) {
- int keyLength = splitKey[j].length;
- byte[] newKey = new byte[keyLength + 2];
- ByteBuffer buffer = ByteBuffer.wrap(newKey);
- buffer.putShort((short) keyLength);
- System.arraycopy(splitKey[j], 0, newKey, 2, keyLength);
- noDictionaryColumnsData[j][i] = newKey;
- }
- //complex types
- else {
- // Need to write columnar block from complex byte array
- int index = complexColumnIndex - noDictionaryCount;
- GenericDataType complexDataType = complexIndexMap.get(index);
- complexColumnIndex++;
- if (complexDataType != null) {
- List<ArrayList<byte[]>> columnsArray = new ArrayList<ArrayList<byte[]>>();
- for (int k = 0; k < complexDataType.getColsCount(); k++) {
- columnsArray.add(new ArrayList<byte[]>());
- }
-
- try {
- ByteBuffer byteArrayInput = ByteBuffer.wrap(splitKey[j]);
- ByteArrayOutputStream byteArrayOutput = new ByteArrayOutputStream();
- DataOutputStream dataOutputStream = new DataOutputStream(byteArrayOutput);
- complexDataType
- .parseAndBitPack(byteArrayInput, dataOutputStream, this.complexKeyGenerator);
- complexDataType.getColumnarDataForComplexType(columnsArray,
- ByteBuffer.wrap(byteArrayOutput.toByteArray()));
- byteArrayOutput.close();
- } catch (IOException e) {
- throw new CarbonDataWriterException(
- "Problem while bit packing and writing complex datatype", e);
- } catch (KeyGenException e) {
- throw new CarbonDataWriterException(
- "Problem while bit packing and writing complex datatype", e);
- }
-
- for (ArrayList<byte[]> eachColumn : columnsArray) {
- colsAndValues.get(complexTypeIndex++).addAll(eachColumn);
- }
- } else {
- // This case not possible as ComplexType is the last columns
- }
+
+ // convert noDictionary columns and complex columns.
+ if (noDictionaryCount > 0 || complexColCount > 0) {
+ byte[][] noDictAndComplex = (byte[][])(rows[mdKeyIndex - 1]);
+ for (int i = 0; i < noDictAndComplex.length; i++) {
+ if (i < noDictionaryCount) {
+ // noDictionary columns, since it is variable length, we need to prepare each
+ // element as LV encoded byte array (first two bytes are the length of the array)
+ byte[] valueWithLength = addLengthToByteArray(noDictAndComplex[i]);
+ noDictDimensionPage[i].putByteArray(rowId, valueWithLength);
+ } else {
+ // complex columns
+ addComplexColumn(i - noDictionaryCount, rowId, noDictAndComplex[i]);
}
}
}
- }
- thread_pool_size = Integer.parseInt(CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.NUM_CORES_BLOCK_SORT,
- CarbonCommonConstants.NUM_CORES_BLOCK_SORT_DEFAULT_VAL));
- ExecutorService executorService = Executors.newFixedThreadPool(thread_pool_size);
- List<Future<IndexStorage>> submit = new ArrayList<Future<IndexStorage>>(
- primitiveDimLens.length + noDictionaryCount + complexColCount);
- int i = 0;
- int dictionaryColumnCount = -1;
- int noDictionaryColumnCount = -1;
- boolean isSortColumn = false;
- for (i = 0; i < dimensionType.length; i++) {
- isSortColumn = i < segmentProperties.getNumberOfSortColumns();
- if (dimensionType[i]) {
- dictionaryColumnCount++;
- if (colGrpModel.isColumnar(dictionaryColumnCount)) {
- submit.add(executorService.submit(
- new BlockSortThread(i, dataHolders[dictionaryColumnCount].getData(), isSortColumn,
- isUseInvertedIndex[i] & isSortColumn)));
- } else {
- submit.add(
- executorService.submit(new ColGroupBlockStorage(dataHolders[dictionaryColumnCount])));
+
+ // convert measure columns
+ for (int i = 0; i < type.length; i++) {
+ Object value = rows[i];
+
+ // in compaction flow the measure with decimal type will come as spark decimal.
+ // need to convert it to byte array.
+ if (type[i] == DataType.DECIMAL && compactionFlow) {
+ BigDecimal bigDecimal = ((Decimal) rows[i]).toJavaBigDecimal();
+ value = DataTypeUtil.bigDecimalToByte(bigDecimal);
}
- } else {
- submit.add(executorService.submit(
- new BlockSortThread(i, noDictionaryColumnsData[++noDictionaryColumnCount], false, true,
- isSortColumn, isUseInvertedIndex[i] & isSortColumn)));
+ measurePage[i].putData(rowId, value);
}
}
- for (int k = 0; k < complexColCount; k++) {
- submit.add(executorService.submit(new BlockSortThread(i++,
- colsAndValues.get(k).toArray(new byte[colsAndValues.get(k).size()][]), false, true)));
- }
- executorService.shutdown();
- try {
- executorService.awaitTermination(1, TimeUnit.DAYS);
- } catch (InterruptedException e) {
- LOGGER.error(e, e.getMessage());
- }
- IndexStorage[] blockStorage =
- new IndexStorage[colGrpModel.getNoOfColumnStore() + noDictionaryCount + complexColCount];
- try {
- for (int k = 0; k < blockStorage.length; k++) {
- blockStorage[k] = submit.get(k).get();
+
+ /**
+ * add a complex column into internal member compleDimensionPage
+ * @param index index of the complexDimensionPage
+ * @param rowId Id of the input row
+ * @param complexColumns byte array the complex columm to be added, extracted of input row
+ */
+ // TODO: this function should be refactoried, ColumnPage should support complex type encoding
+ // directly instead of doing it here
+ private void addComplexColumn(int index, int rowId, byte[] complexColumns) {
+ GenericDataType complexDataType = complexIndexMap.get(index + primitiveDimLens.length);
+
+ // initialize the page if first row
+ if (rowId == 0) {
+ int depthInComplexColumn = complexDataType.getColsCount();
+ complexDimensionPage[index] = new ComplexColumnPage(pageSize, depthInComplexColumn);
}
- } catch (Exception e) {
- LOGGER.error(e, e.getMessage());
- }
- byte[] composedNonDictStartKey = null;
- byte[] composedNonDictEndKey = null;
-
- int numberOfDictSortColumns = segmentProperties.getNumberOfDictSortColumns();
- // generate start/end key by sort_columns
- if (numberOfDictSortColumns > 0) {
- // if sort_columns contain dictionary columns
- int[] keySize = columnarSplitter.getBlockKeySize();
- if (keySize.length > numberOfDictSortColumns) {
- int newMdkLength = 0;
- for (int index = 0; index < numberOfDictSortColumns; index++) {
- newMdkLength += keySize[index];
- }
- byte[] newStartKeyOfSortKey = new byte[newMdkLength];
- byte[] newEndKeyOfSortKey = new byte[newMdkLength];
- System.arraycopy(startkeyLocal, 0, newStartKeyOfSortKey, 0, newMdkLength);
- System.arraycopy(endKeyLocal, 0, newEndKeyOfSortKey, 0, newMdkLength);
- startkeyLocal = newStartKeyOfSortKey;
- endKeyLocal = newEndKeyOfSortKey;
+
+ int depthInComplexColumn = complexDimensionPage[index].getDepth();
+ // this is the encoded columnar data which will be added to page,
+ // size of this list is the depth of complex column, we will fill it by input data
+ List<ArrayList<byte[]>> encodedComplexColumnar = new ArrayList<>();
+ for (int k = 0; k < depthInComplexColumn; k++) {
+ encodedComplexColumnar.add(new ArrayList<byte[]>());
+ }
+
+ // encode the complex type data and fill columnsArray
+ try {
+ ByteBuffer byteArrayInput = ByteBuffer.wrap(complexColumns);
+ ByteArrayOutputStream byteArrayOutput = new ByteArrayOutputStream();
+ DataOutputStream dataOutputStream = new DataOutputStream(byteArrayOutput);
+ complexDataType.parseAndBitPack(byteArrayInput, dataOutputStream, complexKeyGenerator);
+ complexDataType.getColumnarDataForComplexType(encodedComplexColumnar,
+ ByteBuffer.wrap(byteArrayOutput.toByteArray()));
+ byteArrayOutput.close();
+ } catch (IOException | KeyGenException e) {
+ throw new CarbonDataWriterException(
+ "Problem while bit packing and writing complex datatype", e);
}
- } else {
- startkeyLocal = new byte[0];
- endKeyLocal = new byte[0];
- }
- int numberOfNoDictSortColumns = segmentProperties.getNumberOfNoDictSortColumns();
- if (numberOfNoDictSortColumns > 0) {
- // if sort_columns contain no-dictionary columns
- if (noDictionaryStartKey.length > numberOfNoDictSortColumns) {
- byte[][] newNoDictionaryStartKey = new byte[numberOfNoDictSortColumns][];
- byte[][] newNoDictionaryEndKey = new byte[numberOfNoDictSortColumns][];
- System.arraycopy(noDictionaryStartKey, 0, newNoDictionaryStartKey, 0,
- numberOfNoDictSortColumns);
- System
- .arraycopy(noDictionaryEndKey, 0, newNoDictionaryEndKey, 0, numberOfNoDictSortColumns);
- noDictionaryStartKey = newNoDictionaryStartKey;
- noDictionaryEndKey = newNoDictionaryEndKey;
+ for (int depth = 0; depth < depthInComplexColumn; depth++) {
+ complexDimensionPage[index].putComplexData(rowId, depth, encodedComplexColumnar.get(depth));
}
- composedNonDictStartKey =
- NonDictionaryUtil.packByteBufferIntoSingleByteArray(noDictionaryStartKey);
- composedNonDictEndKey =
- NonDictionaryUtil.packByteBufferIntoSingleByteArray(noDictionaryEndKey);
}
- return this.dataWriter
- .buildDataNodeHolder(blockStorage, measureArray, entryCountLocal, startkeyLocal,
- endKeyLocal, compressionModel, composedNonDictStartKey, composedNonDictEndKey,
- nullValueIndexBitSet);
+
+ // Adds length as a short element (first 2 bytes) to the head of the input byte array
+ private byte[] addLengthToByteArray(byte[] input) {
+ byte[] output = new byte[input.length + 2];
+ ByteBuffer buffer = ByteBuffer.wrap(output);
+ buffer.putShort((short) input.length);
+ buffer.put(input, 0, input.length);
+ return output;
+ }
+
}
/**
- * DataHolder will have all row mdkey data
- *
- * @param noOfColumn : no of column participated in mdkey
- * @param noOfRow : total no of row
- * @return : dataholder
+ * generate the NodeHolder from the input rows (one page in case of V3 format)
*/
- private DataHolder[] getDataHolders(int noOfColumn, int noOfRow) {
- DataHolder[] dataHolders = new DataHolder[noOfColumn];
- int colGrpId = -1;
- for (int colGrp = 0; colGrp < noOfColumn; colGrp++) {
- if (colGrpModel.isColumnar(colGrp)) {
- dataHolders[colGrp] = new ColumnDataHolder(noOfRow);
- } else {
- ColGroupMinMax colGrpMinMax = new ColGroupMinMax(segmentProperties, ++colGrpId);
- dataHolders[colGrp] =
- new ColGroupDataHolder(this.columnarSplitter.getBlockKeySize()[colGrp], noOfRow,
- colGrpMinMax);
- }
+ private NodeHolder processDataRows(List<Object[]> dataRows)
+ throws CarbonDataWriterException {
+ if (dataRows.size() == 0) {
+ return new NodeHolder();
+ }
+ TablePage tablePage = new TablePage(dataRows.size());
+ IndexKey keys = new IndexKey(dataRows.size());
+ int rowId = 0;
+
+ // convert row to columnar data
+ for (Object[] row : dataRows) {
+ tablePage.addRow(rowId, row);
+ keys.update(rowId, row);
+ rowId++;
+ }
+
+ // encode and compress dimensions and measure
+ // TODO: To make the encoding more transparent to the user, user should be enable to specify
+ // the encoding and compression method for each type when creating table.
+
+ Codec codec = new Codec();
+ IndexStorage[] dimColumns = codec.encodeAndCompressDimensions(tablePage);
+ Codec encodedMeasure = codec.encodeAndCompressMeasures(tablePage);
+
+ // prepare nullBitSet for writer, remove this after writer can accept TablePage
+ BitSet[] nullBitSet = new BitSet[tablePage.measurePage.length];
+ for (int i = 0; i < nullBitSet.length; i++) {
+ nullBitSet[i] = tablePage.measurePage[i].getNullBitSet();
}
- return dataHolders;
+
+ LOGGER.info("Number Of records processed: " + dataRows.size());
+
+ // TODO: writer interface should be modified to use TablePage
+ return dataWriter.buildDataNodeHolder(dimColumns, encodedMeasure.getEncodedMeasure(),
+ dataRows.size(), keys.startKey, keys.endKey, encodedMeasure.getCompressionModel(),
+ keys.packedNoDictStartKey, keys.packedNoDictEndKey, nullBitSet);
}
/**
@@ -959,7 +788,8 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
return count;
}
- private int getComplexColsCount() {
+ // return the number of complex column after complex columns are expanded
+ private int getExpandedComplexColsCount() {
int count = 0;
for (int i = 0; i < dimensionCount; i++) {
GenericDataType complexDataType = complexIndexMap.get(i);
@@ -970,6 +800,11 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
return count;
}
+ // return the number of complex column
+ private int getComplexColumnCount() {
+ return complexIndexMap.size();
+ }
+
/**
* below method will be used to close the handler
*/
@@ -995,80 +830,6 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
}
/**
- * @param value
- * @return it return no of value after decimal
- */
- private int getDecimalCount(double value) {
- String strValue = BigDecimal.valueOf(Math.abs(value)).toPlainString();
- int integerPlaces = strValue.indexOf('.');
- int decimalPlaces = 0;
- if (-1 != integerPlaces) {
- decimalPlaces = strValue.length() - integerPlaces - 1;
- }
- return decimalPlaces;
- }
-
- /**
- * This method will be used to update the max value for each measure
- */
- private void calculateMaxMin(Object[] max, Object[] min, int[] decimal, int[] msrIndex,
- Object[] row) {
- // Update row level min max
- for (int i = 0; i < msrIndex.length; i++) {
- int count = msrIndex[i];
- if (row[count] != null) {
- if (type[count] == CarbonCommonConstants.DOUBLE_MEASURE) {
- double value = (double) row[count];
- double maxVal = (double) max[count];
- double minVal = (double) min[count];
- max[count] = (maxVal > value ? max[count] : value);
- min[count] = (minVal < value ? min[count] : value);
- int num = getDecimalCount(value);
- decimal[count] = (decimal[count] > num ? decimal[count] : num);
- } else if (type[count] == CarbonCommonConstants.BIG_INT_MEASURE) {
- long value = (long) row[count];
- long maxVal = (long) max[count];
- long minVal = (long) min[count];
- max[count] = (maxVal > value ? max[count] : value);
- min[count] = (minVal < value ? min[count] : value);
- } else if (type[count] == CarbonCommonConstants.BIG_DECIMAL_MEASURE) {
- byte[] buff = null;
- // in compaction flow the measure with decimal type will come as spark decimal.
- // need to convert it to byte array.
- if (this.compactionFlow) {
- BigDecimal bigDecimal = ((Decimal) row[count]).toJavaBigDecimal();
- buff = DataTypeUtil.bigDecimalToByte(bigDecimal);
- } else {
- buff = (byte[]) row[count];
- }
- BigDecimal value = DataTypeUtil.byteToBigDecimal(buff);
- decimal[count] = value.scale();
- }
- }
- }
- }
-
- /**
- * This method will calculate the unique value which will be used as storage
- * key for null values of measures
- *
- * @param minValue
- * @param uniqueValue
- */
- private void calculateUniqueValue(Object[] minValue, Object[] uniqueValue) {
- for (int i = 0; i < measureCount; i++) {
- if (type[i] == CarbonCommonConstants.BIG_INT_MEASURE) {
- uniqueValue[i] = (long) minValue[i] - 1;
- } else if (type[i] == CarbonCommonConstants.BIG_DECIMAL_MEASURE) {
- BigDecimal val = (BigDecimal) minValue[i];
- uniqueValue[i] = (val.subtract(new BigDecimal(1.0)));
- } else {
- uniqueValue[i] = (double) minValue[i] - 1;
- }
- }
- }
-
- /**
* Below method will be to configure fact file writing configuration
*
* @throws CarbonDataWriterException
@@ -1122,15 +883,15 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
List<Integer> customMeasureIndexList =
new ArrayList<Integer>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
for (int j = 0; j < type.length; j++) {
- if (type[j] != CarbonCommonConstants.BYTE_VALUE_MEASURE
- && type[j] != CarbonCommonConstants.BIG_DECIMAL_MEASURE) {
+ if (type[j] != DataType.BYTE && type[j] != DataType.DECIMAL) {
otherMeasureIndexList.add(j);
} else {
customMeasureIndexList.add(j);
}
}
- otherMeasureIndex = new int[otherMeasureIndexList.size()];
- customMeasureIndex = new int[customMeasureIndexList.size()];
+
+ int[] otherMeasureIndex = new int[otherMeasureIndexList.size()];
+ int[] customMeasureIndex = new int[customMeasureIndexList.size()];
for (int i = 0; i < otherMeasureIndex.length; i++) {
otherMeasureIndex[i] = otherMeasureIndexList.get(i);
}
@@ -1157,7 +918,7 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
* @return all dimensions cardinality including complex dimension metadata column
*/
private int[] getBlockKeySizeWithComplexTypes(int[] primitiveBlockKeySize) {
- int allColsCount = getComplexColsCount();
+ int allColsCount = getExpandedComplexColsCount();
int[] blockKeySizeWithComplexTypes =
new int[this.colGrpModel.getNoOfColumnStore() + allColsCount];
@@ -1178,52 +939,6 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
return blockKeySizeWithComplexTypes;
}
- private CarbonWriteDataHolder initialiseKeyBlockHolder(int size) {
- CarbonWriteDataHolder keyDataHolder = new CarbonWriteDataHolder();
- keyDataHolder.initialiseByteArrayValuesForKey(size);
- return keyDataHolder;
- }
-
- private CarbonWriteDataHolder initialiseKeyBlockHolderForNonDictionary(int size) {
- CarbonWriteDataHolder keyDataHolder = new CarbonWriteDataHolder();
- keyDataHolder.initialiseByteArrayValuesForNonDictionary(size);
- return keyDataHolder;
- }
-
- private CarbonWriteDataHolder[] initialiseDataHolder(int size) {
- CarbonWriteDataHolder[] dataHolder = new CarbonWriteDataHolder[this.measureCount];
- for (int i = 0; i < otherMeasureIndex.length; i++) {
- dataHolder[otherMeasureIndex[i]] = new CarbonWriteDataHolder();
- if (type[otherMeasureIndex[i]] == CarbonCommonConstants.BIG_INT_MEASURE) {
- dataHolder[otherMeasureIndex[i]].initialiseLongValues(size);
- } else {
- dataHolder[otherMeasureIndex[i]].initialiseDoubleValues(size);
- }
- }
- for (int i = 0; i < customMeasureIndex.length; i++) {
- dataHolder[customMeasureIndex[i]] = new CarbonWriteDataHolder();
- dataHolder[customMeasureIndex[i]].initialiseByteArrayValues(size);
- }
- return dataHolder;
- }
-
- /**
- * Below method will be used to get the bit set array for
- * all the measure, which will store the indexes which are null
- *
- * @param measureCount
- * @return bit set to store null value index
- */
- private BitSet[] getMeasureNullValueIndexBitSet(int measureCount) {
- // creating a bit set for all the measure column
- BitSet[] nullvalueIndexBitset = new BitSet[measureCount];
- for (int i = 0; i < nullvalueIndexBitset.length; i++) {
- // bitset size will be blocklet size
- nullvalueIndexBitset[i] = new BitSet(this.blockletSize);
- }
- return nullvalueIndexBitset;
- }
-
/**
* Below method will be used to get the fact data writer instance
*
@@ -1255,7 +970,7 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
carbonDataWriterVo.setCarbonDataFileAttributes(carbonDataFileAttributes);
carbonDataWriterVo.setDatabaseName(databaseName);
carbonDataWriterVo.setWrapperColumnSchemaList(wrapperColumnSchemaList);
- carbonDataWriterVo.setIsDictionaryColumn(dimensionType);
+ carbonDataWriterVo.setIsDictionaryColumn(isDictDimension);
carbonDataWriterVo.setCarbonDataDirectoryPath(carbonDataDirectoryPath);
carbonDataWriterVo.setColCardinality(colCardinality);
carbonDataWriterVo.setSegmentProperties(segmentProperties);
@@ -1482,13 +1197,185 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
}
} else {
if (version == ColumnarFormatVersion.V3) {
- return new BlockIndexerStorageForNoInvertedIndexForShort(this.data, isNoDictionary);
+ return new BlockIndexerStorageForNoInvertedIndexForShort(this.data);
} else {
- return new BlockIndexerStorageForNoInvertedIndex(this.data, isNoDictionary);
+ return new BlockIndexerStorageForNoInvertedIndex(this.data);
}
}
}
}
+
+ public class Codec {
+ private WriterCompressModel compressionModel;
+ private byte[][] encodedMeasureArray;
+
+ Codec() {
+ }
+
+ public WriterCompressModel getCompressionModel() {
+ return compressionModel;
+ }
+
+ public byte[][] getEncodedMeasure() {
+ return encodedMeasureArray;
+ }
+
+ public Codec encodeAndCompressMeasures(TablePage tablePage) {
+ // TODO: following conversion is required only because compress model requires them,
+ // remove then after the compress framework is refactoried
+ FixLengthColumnPage[] measurePage = tablePage.measurePage;
+ int measureCount = measurePage.length;
+ Object[] min = new Object[measurePage.length];
+ Object[] max = new Object[measurePage.length];
+ Object[] uniqueValue = new Object[measurePage.length];
+ int[] decimal = new int[measurePage.length];
+ for (int i = 0; i < measurePage.length; i++) {
+ min[i] = measurePage[i].getStatistics().getMin();
+ max[i] = measurePage[i].getStatistics().getMax();
+ uniqueValue[i] = measurePage[i].getStatistics().getUniqueValue();
+ decimal[i] = measurePage[i].getStatistics().getDecimal();
+ }
+ // encode and compress measure column page
+ compressionModel = ValueCompressionUtil
+ .getWriterCompressModel(max, min, decimal, uniqueValue, type, new byte[measureCount]);
+ encodedMeasureArray = encodeMeasure(compressionModel, measurePage);
+ return this;
+ }
+
+ // this method first invokes encoding routine to encode the data chunk,
+ // followed by invoking compression routine for preparing the data chunk for writing.
+ private byte[][] encodeMeasure(WriterCompressModel compressionModel,
+ FixLengthColumnPage[] columnPages) {
+
+ CarbonWriteDataHolder[] holders = new CarbonWriteDataHolder[columnPages.length];
+ for (int i = 0; i < holders.length; i++) {
+ holders[i] = new CarbonWriteDataHolder();
+ switch (columnPages[i].getDataType()) {
+ case SHORT:
+ case INT:
+ case LONG:
+ holders[i].setWritableLongPage(columnPages[i].getLongPage());
+ break;
+ case DOUBLE:
+ holders[i].setWritableDoublePage(columnPages[i].getDoublePage());
+ break;
+ case DECIMAL:
+ holders[i].setWritableDecimalPage(columnPages[i].getDecimalPage());
+ break;
+ default:
+ throw new RuntimeException("Unsupported data type: " + columnPages[i].getDataType());
+ }
+ }
+
+ DataType[] dataType = compressionModel.getDataType();
+ ValueCompressionHolder[] values =
+ new ValueCompressionHolder[compressionModel.getValueCompressionHolder().length];
+ byte[][] returnValue = new byte[values.length][];
+ for (int i = 0; i < compressionModel.getValueCompressionHolder().length; i++) {
+ values[i] = compressionModel.getValueCompressionHolder()[i];
+ if (dataType[i] != DataType.DECIMAL) {
+ values[i].setValue(
+ ValueCompressionUtil.getValueCompressor(compressionModel.getCompressionFinders()[i])
+ .getCompressedValues(compressionModel.getCompressionFinders()[i], holders[i],
+ compressionModel.getMaxValue()[i],
+ compressionModel.getMantissa()[i]));
+ } else {
+ values[i].setValue(holders[i].getWritableByteArrayValues());
+ }
+ values[i].compress();
+ returnValue[i] = values[i].getCompressedData();
+ }
+
+ return returnValue;
+ }
+
+ /**
+ * Encode and compress each column page. The work is done using a thread pool.
+ */
+ private IndexStorage[] encodeAndCompressDimensions(TablePage tablePage) {
+ int noDictionaryCount = tablePage.noDictDimensionPage.length;
+ int complexColCount = tablePage.complexDimensionPage.length;
+
+ // thread pool size to be used for encoding dimension
+ // each thread will sort the column page data and compress it
+ int thread_pool_size = Integer.parseInt(CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.NUM_CORES_BLOCK_SORT,
+ CarbonCommonConstants.NUM_CORES_BLOCK_SORT_DEFAULT_VAL));
+ ExecutorService executorService = Executors.newFixedThreadPool(thread_pool_size);
+ Callable<IndexStorage> callable;
+ List<Future<IndexStorage>> submit = new ArrayList<Future<IndexStorage>>(
+ primitiveDimLens.length + noDictionaryCount + complexColCount);
+ int i = 0;
+ int dictionaryColumnCount = -1;
+ int noDictionaryColumnCount = -1;
+ int colGrpId = -1;
+ boolean isSortColumn = false;
+ for (i = 0; i < isDictDimension.length; i++) {
+ isSortColumn = i < segmentProperties.getNumberOfSortColumns();
+ if (isDictDimension[i]) {
+ dictionaryColumnCount++;
+ if (colGrpModel.isColumnar(dictionaryColumnCount)) {
+ // dictionary dimension
+ callable =
+ new BlockSortThread(
+ i,
+ tablePage.dictDimensionPage[dictionaryColumnCount].getByteArrayPage(),
+ isSortColumn,
+ isUseInvertedIndex[i] & isSortColumn);
+
+ } else {
+ // column group
+ callable = new ColGroupBlockStorage(
+ segmentProperties,
+ ++colGrpId,
+ tablePage.dictDimensionPage[dictionaryColumnCount].getByteArrayPage());
+ }
+ } else {
+ // no dictionary dimension
+ callable =
+ new BlockSortThread(
+ i,
+ tablePage.noDictDimensionPage[++noDictionaryColumnCount].getByteArrayPage(),
+ false,
+ true,
+ isSortColumn,
+ isUseInvertedIndex[i] & isSortColumn);
+ }
+ // start a thread to sort the page data
+ submit.add(executorService.submit(callable));
+ }
+
+ // complex type column
+ for (int index = 0; index < getComplexColumnCount(); index++) {
+ Iterator<byte[][]> iterator = tablePage.complexDimensionPage[index].iterator();
+ while (iterator.hasNext()) {
+ callable =
+ new BlockSortThread(
+ i++,
+ iterator.next(),
+ false,
+ true);
+ submit.add(executorService.submit(callable));
+ }
+ }
+ executorService.shutdown();
+ try {
+ executorService.awaitTermination(1, TimeUnit.DAYS);
+ } catch (InterruptedException e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ IndexStorage[] dimColumns = new IndexStorage[
+ colGrpModel.getNoOfColumnStore() + noDictionaryCount + getExpandedComplexColsCount()];
+ try {
+ for (int k = 0; k < dimColumns.length; k++) {
+ dimColumns[k] = submit.get(k).get();
+ }
+ } catch (Exception e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ return dimColumns;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
index 6fd29d7..44958a4 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
@@ -18,7 +18,6 @@
package org.apache.carbondata.processing.store;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
@@ -29,13 +28,13 @@ import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.keygenerator.KeyGenerator;
import org.apache.carbondata.core.metadata.CarbonMetadata;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.carbondata.core.util.path.CarbonStorePath;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.processing.datatypes.GenericDataType;
@@ -146,10 +145,9 @@ public class CarbonFactDataHandlerModel {
private int[] primitiveDimLens;
/**
- * array in which each character represents an aggregation type and
- * the array length will be equal to the number of measures in table
+ * data type of all measures in the table
*/
- private char[] aggType;
+ private DataType[] measureDataType;
/**
* carbon data file attributes like task id, file stamp
*/
@@ -283,8 +281,8 @@ public class CarbonFactDataHandlerModel {
carbonFactDataHandlerModel.setSegmentProperties(segmentProperties);
carbonFactDataHandlerModel.setColCardinality(colCardinality);
carbonFactDataHandlerModel.setDataWritingRequest(true);
- carbonFactDataHandlerModel.setAggType(CarbonDataProcessorUtil
- .getAggType(configuration.getMeasureCount(), configuration.getMeasureFields()));
+ carbonFactDataHandlerModel.setMeasureDataType(CarbonDataProcessorUtil
+ .getMeasureDataType(configuration.getMeasureCount(), configuration.getMeasureFields()));
carbonFactDataHandlerModel.setFactDimLens(dimLens);
carbonFactDataHandlerModel.setWrapperColumnSchema(wrapperColumnSchema);
carbonFactDataHandlerModel.setPrimitiveDimLens(simpleDimsLen);
@@ -340,13 +338,12 @@ public class CarbonFactDataHandlerModel {
new HashMap<Integer, GenericDataType>(segmentProperties.getComplexDimensions().size());
carbonFactDataHandlerModel.setComplexIndexMap(complexIndexMap);
carbonFactDataHandlerModel.setDataWritingRequest(true);
- char[] aggType = new char[segmentProperties.getMeasures().size()];
- Arrays.fill(aggType, 'n');
+ DataType[] aggType = new DataType[segmentProperties.getMeasures().size()];
int i = 0;
for (CarbonMeasure msr : segmentProperties.getMeasures()) {
- aggType[i++] = DataTypeUtil.getAggType(msr.getDataType());
+ aggType[i++] = msr.getDataType();
}
- carbonFactDataHandlerModel.setAggType(aggType);
+ carbonFactDataHandlerModel.setMeasureDataType(aggType);
carbonFactDataHandlerModel.setFactDimLens(segmentProperties.getDimColumnsCardinality());
String carbonDataDirectoryPath = CarbonDataProcessorUtil
.checkAndCreateCarbonStoreLocation(loadModel.getStorePath(), loadModel.getDatabaseName(),
@@ -501,12 +498,12 @@ public class CarbonFactDataHandlerModel {
this.primitiveDimLens = primitiveDimLens;
}
- public char[] getAggType() {
- return aggType;
+ public DataType[] getMeasureDataType() {
+ return measureDataType;
}
- public void setAggType(char[] aggType) {
- this.aggType = aggType;
+ public void setMeasureDataType(DataType[] measureDataType) {
+ this.measureDataType = measureDataType;
}
public String getCarbonDataDirectoryPath() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/processing/src/main/java/org/apache/carbondata/processing/store/SingleThreadFinalSortFilesMerger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/SingleThreadFinalSortFilesMerger.java b/processing/src/main/java/org/apache/carbondata/processing/store/SingleThreadFinalSortFilesMerger.java
index f8454f1..6a33f34 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/SingleThreadFinalSortFilesMerger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/SingleThreadFinalSortFilesMerger.java
@@ -30,6 +30,7 @@ import org.apache.carbondata.common.CarbonIterator;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.processing.sortandgroupby.exception.CarbonSortKeyAndGroupByException;
import org.apache.carbondata.processing.sortandgroupby.sortdata.SortTempFileChunkHolder;
@@ -93,7 +94,7 @@ public class SingleThreadFinalSortFilesMerger extends CarbonIterator<Object[]> {
*/
private String tempFileLocation;
- private char[] aggType;
+ private DataType[] measureDataType;
/**
* below code is to check whether dimension
@@ -105,13 +106,13 @@ public class SingleThreadFinalSortFilesMerger extends CarbonIterator<Object[]> {
public SingleThreadFinalSortFilesMerger(String tempFileLocation, String tableName,
int dimensionCount, int complexDimensionCount, int measureCount, int noDictionaryCount,
- char[] aggType, boolean[] isNoDictionaryColumn, boolean[] isNoDictionarySortColumn) {
+ DataType[] type, boolean[] isNoDictionaryColumn, boolean[] isNoDictionarySortColumn) {
this.tempFileLocation = tempFileLocation;
this.tableName = tableName;
this.dimensionCount = dimensionCount;
this.complexDimensionCount = complexDimensionCount;
this.measureCount = measureCount;
- this.aggType = aggType;
+ this.measureDataType = type;
this.noDictionaryCount = noDictionaryCount;
this.isNoDictionaryColumn = isNoDictionaryColumn;
this.isNoDictionarySortColumn = isNoDictionarySortColumn;
@@ -183,8 +184,8 @@ public class SingleThreadFinalSortFilesMerger extends CarbonIterator<Object[]> {
// create chunk holder
SortTempFileChunkHolder sortTempFileChunkHolder =
new SortTempFileChunkHolder(tempFile, dimensionCount, complexDimensionCount,
- measureCount, fileBufferSize, noDictionaryCount, aggType, isNoDictionaryColumn,
- isNoDictionarySortColumn);
+ measureCount, fileBufferSize, noDictionaryCount, measureDataType,
+ isNoDictionaryColumn, isNoDictionarySortColumn);
// initialize
sortTempFileChunkHolder.initialize();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/processing/src/main/java/org/apache/carbondata/processing/store/colgroup/ColGroupBlockStorage.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/colgroup/ColGroupBlockStorage.java b/processing/src/main/java/org/apache/carbondata/processing/store/colgroup/ColGroupBlockStorage.java
index 2049bec..def2aaa 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/colgroup/ColGroupBlockStorage.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/colgroup/ColGroupBlockStorage.java
@@ -14,10 +14,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.carbondata.processing.store.colgroup;
import java.util.concurrent.Callable;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.columnar.IndexStorage;
/**
@@ -25,8 +27,22 @@ import org.apache.carbondata.core.datastore.columnar.IndexStorage;
*/
public class ColGroupBlockStorage implements IndexStorage, Callable<IndexStorage> {
+ private byte[][] data;
+
+ private ColGroupMinMax colGrpMinMax;
+
+ public ColGroupBlockStorage(SegmentProperties segmentProperties, int colGrpIndex, byte[][] data) {
+ colGrpMinMax = new ColGroupMinMax(segmentProperties, colGrpIndex);
+ this.data = data;
+ for (int i = 0; i < data.length; i++) {
+ colGrpMinMax.add(data[i]);
+ }
+ }
+
+ @Deprecated
private ColGroupDataHolder colGrpDataHolder;
+ @Deprecated
public ColGroupBlockStorage(DataHolder colGrpDataHolder) {
this.colGrpDataHolder = (ColGroupDataHolder) colGrpDataHolder;
}
@@ -58,7 +74,7 @@ public class ColGroupBlockStorage implements IndexStorage, Callable<IndexStorage
* for column group storage its not required
*/
@Override public byte[][] getKeyBlock() {
- return colGrpDataHolder.getData();
+ return data;
}
/**
@@ -73,22 +89,19 @@ public class ColGroupBlockStorage implements IndexStorage, Callable<IndexStorage
* for column group storage its not required
*/
@Override public int getTotalSize() {
- return colGrpDataHolder.getTotalSize();
+ return data.length;
}
- /**
- * Get min max of column group storage
- */
@Override public byte[] getMin() {
- return colGrpDataHolder.getMin();
+ return colGrpMinMax.getMin();
}
@Override public byte[] getMax() {
- return colGrpDataHolder.getMax();
+ return colGrpMinMax.getMax();
}
/**
- * return
+ * return self
*/
@Override public IndexStorage call() throws Exception {
return this;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
index bb80d1e..18f1b2e 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
@@ -105,10 +105,8 @@ public class CarbonFactDataWriterImplV3 extends AbstractFactDataWriter<short[]>
int[] keyLengths = new int[keyStorageArray.length];
// below will calculate min and max value for each column
- // for below 2d array, first index will be for column and second will be min
- // max
+ // for below 2d array, first index will be for column and second will be min and max
// value for same column
- // byte[][] columnMinMaxData = new byte[keyStorageArray.length][];
byte[][] dimensionMinValue = new byte[keyStorageArray.length][];
byte[][] dimensionMaxValue = new byte[keyStorageArray.length][];
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
index c6eeb4c..3c090fe 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
@@ -20,7 +20,6 @@ package org.apache.carbondata.processing.util;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -48,7 +47,6 @@ import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.carbondata.core.util.path.CarbonStorePath;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.processing.datatypes.ArrayDataType;
@@ -406,31 +404,28 @@ public final class CarbonDataProcessorUtil {
return columnNames;
}
- /**
- * get agg type
- */
- public static char[] getAggType(int measureCount, String databaseName, String tableName) {
- char[] aggType = new char[measureCount];
- Arrays.fill(aggType, 'n');
+
+ public static DataType[] getMeasureDataType(int measureCount, String databaseName,
+ String tableName) {
+ DataType[] type = new DataType[measureCount];
+ for (int i = 0; i < type.length; i++) {
+ type[i] = DataType.DOUBLE;
+ }
CarbonTable carbonTable = CarbonMetadata.getInstance().getCarbonTable(
databaseName + CarbonCommonConstants.UNDERSCORE + tableName);
List<CarbonMeasure> measures = carbonTable.getMeasureByTableName(tableName);
- for (int i = 0; i < aggType.length; i++) {
- aggType[i] = DataTypeUtil.getAggType(measures.get(i).getDataType());
+ for (int i = 0; i < type.length; i++) {
+ type[i] = measures.get(i).getDataType();
}
- return aggType;
+ return type;
}
- /**
- * get agg type
- */
- public static char[] getAggType(int measureCount, DataField[] measureFields) {
- char[] aggType = new char[measureCount];
- Arrays.fill(aggType, 'n');
- for (int i = 0; i < measureFields.length; i++) {
- aggType[i] = DataTypeUtil.getAggType(measureFields[i].getColumn().getDataType());
+ public static DataType[] getMeasureDataType(int measureCount, DataField[] measureFields) {
+ DataType[] type = new DataType[measureCount];
+ for (int i = 0; i < type.length; i++) {
+ type[i] = measureFields[i].getColumn().getDataType();
}
- return aggType;
+ return type;
}
/**
@@ -509,16 +504,19 @@ public final class CarbonDataProcessorUtil {
}
/**
- * initialise aggregation type for measures for their storage format
+ * initialise data type for measures for their storage format
*/
- public static char[] initAggType(CarbonTable carbonTable, String tableName, int measureCount) {
- char[] aggType = new char[measureCount];
- Arrays.fill(aggType, 'n');
+ public static DataType[] initDataType(CarbonTable carbonTable, String tableName,
+ int measureCount) {
+ DataType[] type = new DataType[measureCount];
+ for (int i = 0; i < type.length; i++) {
+ type[i] = DataType.DOUBLE;
+ }
List<CarbonMeasure> measures = carbonTable.getMeasureByTableName(tableName);
for (int i = 0; i < measureCount; i++) {
- aggType[i] = DataTypeUtil.getAggType(measures.get(i).getDataType());
+ type[i] = measures.get(i).getDataType();
}
- return aggType;
+ return type;
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/processing/src/test/java/org/apache/carbondata/processing/store/colgroup/ColGroupMinMaxTest.java
----------------------------------------------------------------------
diff --git a/processing/src/test/java/org/apache/carbondata/processing/store/colgroup/ColGroupMinMaxTest.java b/processing/src/test/java/org/apache/carbondata/processing/store/colgroup/ColGroupMinMaxTest.java
index da10d7a..038ac03 100644
--- a/processing/src/test/java/org/apache/carbondata/processing/store/colgroup/ColGroupMinMaxTest.java
+++ b/processing/src/test/java/org/apache/carbondata/processing/store/colgroup/ColGroupMinMaxTest.java
@@ -33,6 +33,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Test;
/**
@@ -159,6 +160,7 @@ public class ColGroupMinMaxTest {
}
}
+ @Ignore
@Test
public void testRowStoreMinMax() throws KeyGenException {
[17/19] carbondata git commit: [CARBONDATA-936] Create table with
partition and add test case (12-dev) This closes #896
Posted by ra...@apache.org.
[CARBONDATA-936] Create table with partition and add test case (12-dev) This closes #896
create table with partition
fix comment
fix comments
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/86f6a818
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/86f6a818
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/86f6a818
Branch: refs/heads/master
Commit: 86f6a81811239006dee7db55b19d794b7c373cb8
Parents: 98df130
Author: QiangCai <qi...@qq.com>
Authored: Mon May 8 23:20:13 2017 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Sun May 14 20:40:14 2017 +0800
----------------------------------------------------------------------
.../ThriftWrapperSchemaConverterImpl.java | 79 ++++++++++++
.../core/metadata/schema/PartitionInfo.java | 4 +
.../core/metadata/schema/table/CarbonTable.java | 3 +
.../core/metadata/schema/table/TableSchema.java | 2 +-
.../partition/TestDDLForPartitionTable.scala | 127 +++++++++++++++++++
.../execution/command/carbonTableSchema.scala | 9 ++
.../org/apache/spark/sql/CarbonSqlParser.scala | 38 +++++-
.../spark/sql/parser/CarbonSparkSqlParser.scala | 17 ++-
8 files changed, 268 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/86f6a818/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java b/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
index 3d114ec..2d5f395 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
@@ -25,8 +25,10 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.BucketingInfo;
+import org.apache.carbondata.core.metadata.schema.PartitionInfo;
import org.apache.carbondata.core.metadata.schema.SchemaEvolution;
import org.apache.carbondata.core.metadata.schema.SchemaEvolutionEntry;
+import org.apache.carbondata.core.metadata.schema.partition.PartitionType;
import org.apache.carbondata.core.metadata.schema.table.TableInfo;
import org.apache.carbondata.core.metadata.schema.table.TableSchema;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
@@ -185,6 +187,41 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
return thriftColumnSchema;
}
+ private org.apache.carbondata.format.PartitionType fromWrapperToExternalPartitionType(
+ PartitionType wrapperPartitionType) {
+ if (null == wrapperPartitionType) {
+ return null;
+ }
+ switch (wrapperPartitionType) {
+ case HASH:
+ return org.apache.carbondata.format.PartitionType.HASH;
+ case LIST:
+ return org.apache.carbondata.format.PartitionType.LIST;
+ case RANGE:
+ return org.apache.carbondata.format.PartitionType.RANGE;
+ case RANGE_INTERVAL:
+ return org.apache.carbondata.format.PartitionType.RANGE_INTERVAL;
+ default:
+ return org.apache.carbondata.format.PartitionType.HASH;
+ }
+ }
+
+ private org.apache.carbondata.format.PartitionInfo fromWrapperToExternalPartitionInfo(
+ PartitionInfo wrapperPartitionInfo) {
+ List<org.apache.carbondata.format.ColumnSchema> thriftColumnSchema =
+ new ArrayList<org.apache.carbondata.format.ColumnSchema>();
+ for (ColumnSchema wrapperColumnSchema : wrapperPartitionInfo.getColumnSchemaList()) {
+ thriftColumnSchema.add(fromWrapperToExternalColumnSchema(wrapperColumnSchema));
+ }
+ org.apache.carbondata.format.PartitionInfo externalPartitionInfo =
+ new org.apache.carbondata.format.PartitionInfo(thriftColumnSchema,
+ fromWrapperToExternalPartitionType(wrapperPartitionInfo.getPartitionType()));
+ externalPartitionInfo.setList_info(wrapperPartitionInfo.getListInfo());
+ externalPartitionInfo.setRange_info(wrapperPartitionInfo.getRangeInfo());
+ externalPartitionInfo.setNum_partitions(wrapperPartitionInfo.getNumPartitions());
+ return externalPartitionInfo;
+ }
+
/* (non-Javadoc)
* convert from wrapper to external tableschema
*/
@@ -206,6 +243,10 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
externalTableSchema.setBucketingInfo(
fromWrapperToExternalBucketingInfo(wrapperTableSchema.getBucketingInfo()));
}
+ if (wrapperTableSchema.getPartitionInfo() != null) {
+ externalTableSchema.setPartitionInfo(
+ fromWrapperToExternalPartitionInfo(wrapperTableSchema.getPartitionInfo()));
+ }
return externalTableSchema;
}
@@ -381,6 +422,40 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
return wrapperColumnSchema;
}
+ private PartitionType fromExternalToWrapperPartitionType(
+ org.apache.carbondata.format.PartitionType externalPartitionType) {
+ if (null == externalPartitionType) {
+ return null;
+ }
+ switch (externalPartitionType) {
+ case HASH:
+ return PartitionType.HASH;
+ case LIST:
+ return PartitionType.LIST;
+ case RANGE:
+ return PartitionType.RANGE;
+ case RANGE_INTERVAL:
+ return PartitionType.RANGE_INTERVAL;
+ default:
+ return PartitionType.HASH;
+ }
+ }
+
+ private PartitionInfo fromExternalToWrapperPartitionInfo(
+ org.apache.carbondata.format.PartitionInfo externalPartitionInfo) {
+ List<ColumnSchema> wrapperColumnSchema = new ArrayList<ColumnSchema>();
+ for (org.apache.carbondata.format.ColumnSchema columnSchema :
+ externalPartitionInfo.getPartition_columns()) {
+ wrapperColumnSchema.add(fromExternalToWrapperColumnSchema(columnSchema));
+ }
+ PartitionInfo wrapperPartitionInfo = new PartitionInfo(wrapperColumnSchema,
+ fromExternalToWrapperPartitionType(externalPartitionInfo.getPartition_type()));
+ wrapperPartitionInfo.setListInfo(externalPartitionInfo.getList_info());
+ wrapperPartitionInfo.setRangeInfo(externalPartitionInfo.getRange_info());
+ wrapperPartitionInfo.setNumPartitions(externalPartitionInfo.getNum_partitions());
+ return wrapperPartitionInfo;
+ }
+
/* (non-Javadoc)
* convert from external to wrapper tableschema
*/
@@ -402,6 +477,10 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
wrapperTableSchema.setBucketingInfo(
fromExternalToWarpperBucketingInfo(externalTableSchema.bucketingInfo));
}
+ if (externalTableSchema.getPartitionInfo() != null) {
+ wrapperTableSchema.setPartitionInfo(
+ fromExternalToWrapperPartitionInfo(externalTableSchema.getPartitionInfo()));
+ }
return wrapperTableSchema;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/86f6a818/core/src/main/java/org/apache/carbondata/core/metadata/schema/PartitionInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/PartitionInfo.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/PartitionInfo.java
index cd4ac0e..2b08536 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/PartitionInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/PartitionInfo.java
@@ -56,6 +56,10 @@ public class PartitionInfo implements Serializable {
return columnSchemaList;
}
+ public void setColumnSchemaList(List<ColumnSchema> columnSchemaList) {
+ this.columnSchemaList = columnSchemaList;
+ }
+
public PartitionType getPartitionType() {
return partitionType;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/86f6a818/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index ce5cbe4..9f8beb0 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -170,9 +170,12 @@ public class CarbonTable implements Serializable {
this.aggregateTablesName.add(aggTable.getTableName());
fillDimensionsAndMeasuresForTables(aggTable);
tableBucketMap.put(aggTable.getTableName(), aggTable.getBucketingInfo());
+ tablePartitionMap.put(aggTable.getTableName(), aggTable.getPartitionInfo());
}
tableBucketMap.put(tableInfo.getFactTable().getTableName(),
tableInfo.getFactTable().getBucketingInfo());
+ tablePartitionMap.put(tableInfo.getFactTable().getTableName(),
+ tableInfo.getFactTable().getPartitionInfo());
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/86f6a818/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
index ad2730b..f9d848e 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
@@ -195,7 +195,7 @@ public class TableSchema implements Serializable {
return partitionInfo;
}
- public void setpartitionInfo(PartitionInfo partitionInfo) {
+ public void setPartitionInfo(PartitionInfo partitionInfo) {
this.partitionInfo = partitionInfo;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/86f6a818/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDDLForPartitionTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDDLForPartitionTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDDLForPartitionTable.scala
new file mode 100644
index 0000000..00c4df8
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDDLForPartitionTable.scala
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.spark.testsuite.partition
+
+import org.apache.spark.sql.common.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.metadata.CarbonMetadata
+import org.apache.carbondata.core.metadata.datatype.DataType
+import org.apache.carbondata.core.metadata.encoder.Encoding
+import org.apache.carbondata.core.metadata.schema.partition.PartitionType
+
+class TestDDLForPartitionTable extends QueryTest with BeforeAndAfterAll {
+
+ override def beforeAll = {
+ dropTable
+ }
+
+ test("create partition table: hash partition") {
+ sql(
+ """
+ | CREATE TABLE default.hashTable (empname String, designation String, doj Timestamp,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (empno int)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('PARTITION_TYPE'='HASH','NUM_PARTITIONS'='3')
+ """.stripMargin)
+
+ val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default_hashTable")
+ val partitionInfo = carbonTable.getPartitionInfo(carbonTable.getFactTableName)
+ assert(partitionInfo != null)
+ assert(partitionInfo.getColumnSchemaList.get(0).getColumnName.equalsIgnoreCase("empno"))
+ assert(partitionInfo.getColumnSchemaList.get(0).getDataType == DataType.INT)
+ assert(partitionInfo.getColumnSchemaList.get(0).getEncodingList.size == 0)
+ assert(partitionInfo.getPartitionType == PartitionType.HASH)
+ assert(partitionInfo.getNumPartitions == 3)
+ }
+
+ test("create partition table: range partition") {
+ sql(
+ """
+ | CREATE TABLE default.rangeTable (empno int, empname String, designation String,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (doj Timestamp)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('PARTITION_TYPE'='RANGE',
+ | 'RANGE_INFO'='01-01-2010, 01-01-2015, 01-04-2015, 01-07-2015')
+ """.stripMargin)
+
+ val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default_rangeTable")
+ val partitionInfo = carbonTable.getPartitionInfo(carbonTable.getFactTableName)
+ assert(partitionInfo != null)
+ assert(partitionInfo.getColumnSchemaList.get(0).getColumnName.equalsIgnoreCase("doj"))
+ assert(partitionInfo.getColumnSchemaList.get(0).getDataType == DataType.TIMESTAMP)
+ assert(partitionInfo.getColumnSchemaList.get(0).getEncodingList.size == 3)
+ assert(partitionInfo.getColumnSchemaList.get(0).getEncodingList.get(0) == Encoding.DICTIONARY)
+ assert(partitionInfo.getColumnSchemaList.get(0).getEncodingList.get(1) == Encoding.DIRECT_DICTIONARY)
+ assert(partitionInfo.getColumnSchemaList.get(0).getEncodingList.get(2) == Encoding.INVERTED_INDEX)
+ assert(partitionInfo.getPartitionType == PartitionType.RANGE)
+ assert(partitionInfo.getRangeInfo.size == 4)
+ assert(partitionInfo.getRangeInfo.get(0).equals("01-01-2010"))
+ assert(partitionInfo.getRangeInfo.get(1).equals("01-01-2015"))
+ assert(partitionInfo.getRangeInfo.get(2).equals("01-04-2015"))
+ assert(partitionInfo.getRangeInfo.get(3).equals("01-07-2015"))
+ }
+
+ test("create partition table: list partition") {
+ sql(
+ """
+ | CREATE TABLE default.listTable (empno int, empname String, designation String, doj Timestamp,
+ | workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (workgroupcategory string)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('PARTITION_TYPE'='LIST',
+ | 'LIST_INFO'='0, 1, (2, 3)')
+ """.stripMargin)
+ val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default_listTable")
+ val partitionInfo = carbonTable.getPartitionInfo(carbonTable.getFactTableName)
+ assert(partitionInfo != null)
+ assert(partitionInfo.getColumnSchemaList.get(0).getColumnName.equalsIgnoreCase("workgroupcategory"))
+ assert(partitionInfo.getColumnSchemaList.get(0).getDataType == DataType.STRING)
+ assert(partitionInfo.getColumnSchemaList.get(0).getEncodingList.size == 2)
+ assert(partitionInfo.getColumnSchemaList.get(0).getEncodingList.get(0) == Encoding.DICTIONARY)
+ assert(partitionInfo.getColumnSchemaList.get(0).getEncodingList.get(1) == Encoding.INVERTED_INDEX)
+ assert(partitionInfo.getPartitionType == PartitionType.LIST)
+ assert(partitionInfo.getListInfo.size == 3)
+ assert(partitionInfo.getListInfo.get(0).size == 1)
+ assert(partitionInfo.getListInfo.get(0).get(0).equals("0"))
+ assert(partitionInfo.getListInfo.get(1).size == 1)
+ assert(partitionInfo.getListInfo.get(1).get(0).equals("1"))
+ assert(partitionInfo.getListInfo.get(2).size == 2)
+ assert(partitionInfo.getListInfo.get(2).get(0).equals("2"))
+ assert(partitionInfo.getListInfo.get(2).get(1).equals("3"))
+ }
+
+ override def afterAll = {
+ dropTable
+ }
+
+ def dropTable = {
+ sql("drop table if exists hashTable")
+ sql("drop table if exists rangeTable")
+ sql("drop table if exists listTable")
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/86f6a818/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index 6f50ecc..a3892af 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -506,6 +506,15 @@ class TableNewProcessor(cm: TableModel) {
tableSchema.setBucketingInfo(
new BucketingInfo(bucketCols.asJava, cm.bucketFields.get.numberOfBuckets))
}
+ if (cm.partitionInfo.isDefined) {
+ val partitionInfo = cm.partitionInfo.get
+ val PartitionColumnSchema = partitionInfo.getColumnSchemaList.asScala
+ val partitionCols = allColumns.filter { column =>
+ PartitionColumnSchema.exists(_.getColumnName.equalsIgnoreCase(column.getColumnName))
+ }.asJava
+ partitionInfo.setColumnSchemaList(partitionCols)
+ tableSchema.setPartitionInfo(partitionInfo)
+ }
tableSchema.setTableName(cm.tableName)
tableSchema.setListOfColumns(allColumns.asJava)
tableSchema.setSchemaEvalution(schemaEvol)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/86f6a818/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
index 85d54d0..99a20b4 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
@@ -31,6 +31,7 @@ import org.apache.spark.sql.execution.ExplainCommand
import org.apache.spark.sql.execution.command._
import org.apache.spark.sql.execution.datasources.DescribeCommand
import org.apache.spark.sql.hive.HiveQlWrapper
+import org.apache.spark.sql.types.StructField
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
import org.apache.carbondata.spark.util.CommonUtil
@@ -153,6 +154,7 @@ class CarbonSqlParser() extends CarbonDDLSqlParser {
var fields: Seq[Field] = Seq[Field]()
var tableComment: String = ""
var tableProperties = Map[String, String]()
+ var partitionByFields: Seq[Field] = Seq[Field]()
var partitionCols: Seq[PartitionerField] = Seq[PartitionerField]()
var likeTableName: String = ""
var storedBy: String = ""
@@ -239,8 +241,27 @@ class CarbonSqlParser() extends CarbonDDLSqlParser {
val columnName = col.getName()
val dataType = Option(col.getType)
val comment = col.getComment
+ val rawSchema = '`' + col.getName + '`' + ' ' + col.getType
+ val field = Field(columnName, dataType, Some(columnName), None)
+
+ // the data type of the decimal type will be like decimal(10,0)
+ // so checking the start of the string and taking the precision and scale.
+ // resetting the data type with decimal
+ if (field.dataType.getOrElse("").startsWith("decimal")) {
+ val (precision, scale) = getScaleAndPrecision(col.getType)
+ field.precision = precision
+ field.scale = scale
+ field.dataType = Some("decimal")
+ }
+ if (field.dataType.getOrElse("").startsWith("char")) {
+ field.dataType = Some("char")
+ } else if (field.dataType.getOrElse("").startsWith("float")) {
+ field.dataType = Some("float")
+ }
+ field.rawSchema = rawSchema
val partitionCol = new PartitionerField(columnName, dataType, comment)
partitionCols ++= Seq(partitionCol)
+ partitionByFields ++= Seq(field)
}
}
case Token("TOK_TABLEPROPERTIES", list :: Nil) =>
@@ -266,11 +287,26 @@ class CarbonSqlParser() extends CarbonDDLSqlParser {
case _ => // Unsupport features
}
-
// validate tblProperties
if (!CommonUtil.validateTblProperties(tableProperties, fields)) {
throw new MalformedCarbonCommandException("Invalid table properties")
}
+
+ if (partitionCols.nonEmpty) {
+ if (!CommonUtil.validatePartitionColumns(tableProperties, partitionCols)) {
+ throw new MalformedCarbonCommandException("Invalid table properties")
+ }
+ // partition columns should not be part of the schema
+ val colNames = fields.map(_.column)
+ val badPartCols = partitionCols.map(_.partitionColumn).toSet.intersect(colNames.toSet)
+ if (badPartCols.nonEmpty) {
+ throw new MalformedCarbonCommandException(
+ "Partition columns should not be specified in the schema: " +
+ badPartCols.map("\"" + _ + "\"").mkString("[", ",", "]"))
+ }
+ fields ++= partitionByFields
+ }
+
// prepare table model of the collected tokens
val tableModel: TableModel = prepareTableModel(ifNotExistPresent,
dbName,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/86f6a818/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
index 2dc5f66..076af46 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
@@ -16,7 +16,6 @@
*/
package org.apache.spark.sql.parser
-import scala.collection.JavaConverters._
import scala.collection.mutable
import org.apache.spark.sql.catalyst.parser.{AbstractSqlParser, ParseException, SqlBaseParser}
@@ -97,9 +96,10 @@ class CarbonSqlAstBuilder(conf: SQLConf) extends SparkSqlAstBuilder(conf) {
if (ctx.bucketSpec != null) {
operationNotAllowed("CREATE TABLE ... CLUSTERED BY", ctx)
}
- val partitionerFields = Option(ctx.partitionColumns).toSeq.flatMap(visitColTypeList)
- .map( structField =>
- PartitionerField(structField.name, Some(structField.dataType.toString), null))
+ val partitionByStructFields = Option(ctx.partitionColumns).toSeq.flatMap(visitColTypeList)
+ val partitionerFields = partitionByStructFields.map { structField =>
+ PartitionerField(structField.name, Some(structField.dataType.toString), null)
+ }
val cols = Option(ctx.columns).toSeq.flatMap(visitColTypeList)
val properties = Option(ctx.tablePropertyList).map(visitPropertyKeyValues)
.getOrElse(Map.empty)
@@ -122,15 +122,14 @@ class CarbonSqlAstBuilder(conf: SQLConf) extends SparkSqlAstBuilder(conf) {
if (!CommonUtil.validatePartitionColumns(tableProperties, partitionerFields)) {
throw new MalformedCarbonCommandException("Invalid partition definition")
}
- // partition columns must be part of the schema
+ // partition columns should not be part of the schema
val badPartCols = partitionerFields.map(_.partitionColumn).toSet.intersect(colNames.toSet)
- if (badPartCols.isEmpty) {
- operationNotAllowed(s"Partition columns must be specified in the schema: " +
+ if (badPartCols.nonEmpty) {
+ operationNotAllowed(s"Partition columns should not be specified in the schema: " +
badPartCols.map("\"" + _ + "\"").mkString("[", ",", "]"), ctx)
}
}
-
- val fields = parser.getFields(cols)
+ val fields = parser.getFields(cols ++ partitionByStructFields)
val options = new CarbonOption(properties)
// validate tblProperties
val bucketFields = parser.getBucketFields(tableProperties, fields, options)
[13/19] carbondata git commit: [CARBONDATA-1015] Refactory write step
and add ColumnPage in data load This closes #852
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
index eb134d8..2dc5f66 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
@@ -131,7 +131,6 @@ class CarbonSqlAstBuilder(conf: SQLConf) extends SparkSqlAstBuilder(conf) {
}
val fields = parser.getFields(cols)
-
val options = new CarbonOption(properties)
// validate tblProperties
val bucketFields = parser.getBucketFields(tableProperties, fields, options)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
index 81ee408..690f6ef 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
@@ -24,6 +24,7 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
@@ -43,6 +44,8 @@ import org.apache.carbondata.processing.store.SingleThreadFinalSortFilesMerger;
import org.apache.carbondata.processing.store.writer.exception.CarbonDataWriterException;
import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
+import org.apache.spark.sql.types.Decimal;
+
/**
* This class will process the query result and convert the data
* into a format compatible for data load
@@ -89,7 +92,7 @@ public class CompactionResultSortProcessor extends AbstractResultProcessor {
/**
* agg type defined for measures
*/
- private char[] aggType;
+ private DataType[] aggType;
/**
* segment id
*/
@@ -243,14 +246,14 @@ public class CompactionResultSortProcessor extends AbstractResultProcessor {
* This method will convert the spark decimal to java big decimal type
*
* @param value
- * @param aggType
+ * @param type
* @return
*/
- private Object getConvertedMeasureValue(Object value, char aggType) {
- switch (aggType) {
- case CarbonCommonConstants.BIG_DECIMAL_MEASURE:
+ private Object getConvertedMeasureValue(Object value, DataType type) {
+ switch (type) {
+ case DECIMAL:
if (value != null) {
- value = ((org.apache.spark.sql.types.Decimal) value).toJavaBigDecimal();
+ value = ((Decimal) value).toJavaBigDecimal();
}
return value;
default:
@@ -404,6 +407,6 @@ public class CompactionResultSortProcessor extends AbstractResultProcessor {
* initialise aggregation type for measures for their storage format
*/
private void initAggType() {
- aggType = CarbonDataProcessorUtil.initAggType(carbonTable, tableName, measureCount);
+ aggType = CarbonDataProcessorUtil.initDataType(carbonTable, tableName, measureCount);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterImpl.java
index 0e14660..c1aafcd 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterImpl.java
@@ -82,7 +82,7 @@ public class ParallelReadMergeSorterImpl extends AbstractMergeSorter {
new SingleThreadFinalSortFilesMerger(dataFolderLocation, sortParameters.getTableName(),
sortParameters.getDimColCount(),
sortParameters.getComplexDimColCount(), sortParameters.getMeasureColCount(),
- sortParameters.getNoDictionaryCount(), sortParameters.getAggType(),
+ sortParameters.getNoDictionaryCount(), sortParameters.getMeasureDataType(),
sortParameters.getNoDictionaryDimnesionColumn(),
sortParameters.getNoDictionarySortColumn());
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterWithBucketingImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterWithBucketingImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterWithBucketingImpl.java
index 4a70710..bae037d 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterWithBucketingImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterWithBucketingImpl.java
@@ -146,7 +146,7 @@ public class ParallelReadMergeSorterWithBucketingImpl extends AbstractMergeSorte
return new SingleThreadFinalSortFilesMerger(dataFolderLocation, sortParameters.getTableName(),
sortParameters.getDimColCount(), sortParameters.getComplexDimColCount(),
sortParameters.getMeasureColCount(), sortParameters.getNoDictionaryCount(),
- sortParameters.getAggType(), sortParameters.getNoDictionaryDimnesionColumn(),
+ sortParameters.getMeasureDataType(), sortParameters.getNoDictionaryDimnesionColumn(),
this.sortParameters.getNoDictionarySortColumn());
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeCarbonRowPage.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeCarbonRowPage.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeCarbonRowPage.java
index 44f11f7..24109e4 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeCarbonRowPage.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeCarbonRowPage.java
@@ -22,9 +22,9 @@ import java.io.IOException;
import java.math.BigDecimal;
import java.util.Arrays;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.memory.CarbonUnsafe;
import org.apache.carbondata.core.memory.MemoryBlock;
+import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.util.DataTypeUtil;
/**
@@ -40,7 +40,7 @@ public class UnsafeCarbonRowPage {
private int measureSize;
- private char[] aggType;
+ private DataType[] measureDataType;
private long[] nullSetWords;
@@ -55,13 +55,13 @@ public class UnsafeCarbonRowPage {
private boolean saveToDisk;
public UnsafeCarbonRowPage(boolean[] noDictionaryDimensionMapping,
- boolean[] noDictionarySortColumnMapping, int dimensionSize, int measureSize, char[] aggType,
+ boolean[] noDictionarySortColumnMapping, int dimensionSize, int measureSize, DataType[] type,
MemoryBlock memoryBlock, boolean saveToDisk) {
this.noDictionaryDimensionMapping = noDictionaryDimensionMapping;
this.noDictionarySortColumnMapping = noDictionarySortColumnMapping;
this.dimensionSize = dimensionSize;
this.measureSize = measureSize;
- this.aggType = aggType;
+ this.measureDataType = type;
this.saveToDisk = saveToDisk;
this.nullSetWords = new long[((measureSize - 1) >> 6) + 1];
buffer = new IntPointerBuffer(memoryBlock);
@@ -116,24 +116,30 @@ public class UnsafeCarbonRowPage {
for (int mesCount = 0; mesCount < measureSize; mesCount++) {
Object value = row[mesCount + dimensionSize];
if (null != value) {
- if (aggType[mesCount] == CarbonCommonConstants.DOUBLE_MEASURE) {
- Double val = (Double) value;
- CarbonUnsafe.unsafe.putDouble(baseObject, address + size, val);
- size += 8;
- } else if (aggType[mesCount] == CarbonCommonConstants.BIG_INT_MEASURE) {
- Long val = (Long) value;
- CarbonUnsafe.unsafe.putLong(baseObject, address + size, val);
- size += 8;
- } else if (aggType[mesCount] == CarbonCommonConstants.BIG_DECIMAL_MEASURE) {
- BigDecimal val = (BigDecimal) value;
- byte[] bigDecimalInBytes = DataTypeUtil.bigDecimalToByte(val);
- CarbonUnsafe.unsafe.putShort(baseObject, address + size,
- (short) bigDecimalInBytes.length);
- size += 2;
- CarbonUnsafe.unsafe
- .copyMemory(bigDecimalInBytes, CarbonUnsafe.BYTE_ARRAY_OFFSET, baseObject,
- address + size, bigDecimalInBytes.length);
- size += bigDecimalInBytes.length;
+ switch (measureDataType[mesCount]) {
+ case SHORT:
+ case INT:
+ case LONG:
+ Long val = (Long) value;
+ CarbonUnsafe.unsafe.putLong(baseObject, address + size, val);
+ size += 8;
+ break;
+ case DOUBLE:
+ Double doubleVal = (Double) value;
+ CarbonUnsafe.unsafe.putDouble(baseObject, address + size, doubleVal);
+ size += 8;
+ break;
+ case DECIMAL:
+ BigDecimal decimalVal = (BigDecimal) value;
+ byte[] bigDecimalInBytes = DataTypeUtil.bigDecimalToByte(decimalVal);
+ CarbonUnsafe.unsafe.putShort(baseObject, address + size,
+ (short) bigDecimalInBytes.length);
+ size += 2;
+ CarbonUnsafe.unsafe
+ .copyMemory(bigDecimalInBytes, CarbonUnsafe.BYTE_ARRAY_OFFSET, baseObject,
+ address + size, bigDecimalInBytes.length);
+ size += bigDecimalInBytes.length;
+ break;
}
set(nullSetWords, mesCount);
} else {
@@ -187,22 +193,28 @@ public class UnsafeCarbonRowPage {
for (int mesCount = 0; mesCount < measureSize; mesCount++) {
if (isSet(nullSetWords, mesCount)) {
- if (aggType[mesCount] == CarbonCommonConstants.DOUBLE_MEASURE) {
- Double val = CarbonUnsafe.unsafe.getDouble(baseObject, address + size);
- size += 8;
- rowToFill[dimensionSize + mesCount] = val;
- } else if (aggType[mesCount] == CarbonCommonConstants.BIG_INT_MEASURE) {
- Long val = CarbonUnsafe.unsafe.getLong(baseObject, address + size);
- size += 8;
- rowToFill[dimensionSize + mesCount] = val;
- } else if (aggType[mesCount] == CarbonCommonConstants.BIG_DECIMAL_MEASURE) {
- short aShort = CarbonUnsafe.unsafe.getShort(baseObject, address + size);
- byte[] bigDecimalInBytes = new byte[aShort];
- size += 2;
- CarbonUnsafe.unsafe.copyMemory(baseObject, address + size, bigDecimalInBytes,
- CarbonUnsafe.BYTE_ARRAY_OFFSET, bigDecimalInBytes.length);
- size += bigDecimalInBytes.length;
- rowToFill[dimensionSize + mesCount] = bigDecimalInBytes;
+ switch (measureDataType[mesCount]) {
+ case SHORT:
+ case INT:
+ case LONG:
+ Long val = CarbonUnsafe.unsafe.getLong(baseObject, address + size);
+ size += 8;
+ rowToFill[dimensionSize + mesCount] = val;
+ break;
+ case DOUBLE:
+ Double doubleVal = CarbonUnsafe.unsafe.getDouble(baseObject, address + size);
+ size += 8;
+ rowToFill[dimensionSize + mesCount] = doubleVal;
+ break;
+ case DECIMAL:
+ short aShort = CarbonUnsafe.unsafe.getShort(baseObject, address + size);
+ byte[] bigDecimalInBytes = new byte[aShort];
+ size += 2;
+ CarbonUnsafe.unsafe.copyMemory(baseObject, address + size, bigDecimalInBytes,
+ CarbonUnsafe.BYTE_ARRAY_OFFSET, bigDecimalInBytes.length);
+ size += bigDecimalInBytes.length;
+ rowToFill[dimensionSize + mesCount] = bigDecimalInBytes;
+ break;
}
} else {
rowToFill[dimensionSize + mesCount] = null;
@@ -258,33 +270,34 @@ public class UnsafeCarbonRowPage {
for (int mesCount = 0; mesCount < measureSize; mesCount++) {
if (isSet(nullSetWords, mesCount)) {
- if (aggType[mesCount] == CarbonCommonConstants.DOUBLE_MEASURE) {
- double val = CarbonUnsafe.unsafe.getDouble(baseObject, address + size);
- size += 8;
- stream.writeDouble(val);
- } else if (aggType[mesCount] == CarbonCommonConstants.BIG_INT_MEASURE) {
- long val = CarbonUnsafe.unsafe.getLong(baseObject, address + size);
- size += 8;
- stream.writeLong(val);
- } else if (aggType[mesCount] == CarbonCommonConstants.BIG_DECIMAL_MEASURE) {
- short aShort = CarbonUnsafe.unsafe.getShort(baseObject, address + size);
- byte[] bigDecimalInBytes = new byte[aShort];
- size += 2;
- CarbonUnsafe.unsafe.copyMemory(baseObject, address + size, bigDecimalInBytes,
- CarbonUnsafe.BYTE_ARRAY_OFFSET, bigDecimalInBytes.length);
- size += bigDecimalInBytes.length;
- stream.writeShort(aShort);
- stream.write(bigDecimalInBytes);
+ switch (measureDataType[mesCount]) {
+ case SHORT:
+ case INT:
+ case LONG:
+ long val = CarbonUnsafe.unsafe.getLong(baseObject, address + size);
+ size += 8;
+ stream.writeLong(val);
+ break;
+ case DOUBLE:
+ double doubleVal = CarbonUnsafe.unsafe.getDouble(baseObject, address + size);
+ size += 8;
+ stream.writeDouble(doubleVal);
+ break;
+ case DECIMAL:
+ short aShort = CarbonUnsafe.unsafe.getShort(baseObject, address + size);
+ byte[] bigDecimalInBytes = new byte[aShort];
+ size += 2;
+ CarbonUnsafe.unsafe.copyMemory(baseObject, address + size, bigDecimalInBytes,
+ CarbonUnsafe.BYTE_ARRAY_OFFSET, bigDecimalInBytes.length);
+ size += bigDecimalInBytes.length;
+ stream.writeShort(aShort);
+ stream.write(bigDecimalInBytes);
+ break;
}
}
}
}
- private Object[] getRow(long address) {
- Object[] row = new Object[dimensionSize + measureSize];
- return getRow(address, row);
- }
-
public void freeMemory() {
buffer.freeMemory();
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeSortDataRows.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeSortDataRows.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeSortDataRows.java
index 1219910..0acc6a1 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeSortDataRows.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeSortDataRows.java
@@ -115,7 +115,7 @@ public class UnsafeSortDataRows {
this.rowPage = new UnsafeCarbonRowPage(parameters.getNoDictionaryDimnesionColumn(),
parameters.getNoDictionarySortColumn(),
parameters.getDimColCount() + parameters.getComplexDimColCount(),
- parameters.getMeasureColCount(), parameters.getAggType(), baseBlock,
+ parameters.getMeasureColCount(), parameters.getMeasureDataType(), baseBlock,
!UnsafeMemoryManager.INSTANCE.isMemoryAvailable());
// Delete if any older file exists in sort temp folder
deleteSortLocationIfExists();
@@ -178,10 +178,14 @@ public class UnsafeSortDataRows {
dataSorterAndWriterExecutorService.submit(new DataSorterAndWriter(rowPage));
MemoryBlock memoryBlock = getMemoryBlock(inMemoryChunkSize);
boolean saveToDisk = !UnsafeMemoryManager.INSTANCE.isMemoryAvailable();
- rowPage = new UnsafeCarbonRowPage(parameters.getNoDictionaryDimnesionColumn(),
+ rowPage = new UnsafeCarbonRowPage(
+ parameters.getNoDictionaryDimnesionColumn(),
parameters.getNoDictionarySortColumn(),
parameters.getDimColCount() + parameters.getComplexDimColCount(),
- parameters.getMeasureColCount(), parameters.getAggType(), memoryBlock, saveToDisk);
+ parameters.getMeasureColCount(),
+ parameters.getMeasureDataType(),
+ memoryBlock,
+ saveToDisk);
bytesAdded += rowPage.addRow(rowBatch[i]);
} catch (Exception e) {
LOGGER.error(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
index cfdb69a..aee4e51 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
@@ -32,6 +32,7 @@ import java.util.concurrent.Future;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.processing.newflow.sort.unsafe.UnsafeCarbonRowPage;
@@ -122,7 +123,7 @@ public class UnsafeSortTempFileChunkHolder implements SortTempChunkHolder {
private int noDictionaryCount;
- private char[] aggType;
+ private DataType[] measureDataType;
private int numberOfObjectRead;
/**
@@ -150,7 +151,7 @@ public class UnsafeSortTempFileChunkHolder implements SortTempChunkHolder {
// set mdkey length
this.fileBufferSize = parameters.getFileBufferSize();
this.executorService = Executors.newFixedThreadPool(1);
- this.aggType = parameters.getAggType();
+ this.measureDataType = parameters.getMeasureDataType();
this.isNoDictionaryDimensionColumn = parameters.getNoDictionaryDimnesionColumn();
this.nullSetWordsLength = ((measureCount - 1) >> 6) + 1;
comparator = new NewRowComparator(parameters.getNoDictionarySortColumn());
@@ -323,15 +324,21 @@ public class UnsafeSortTempFileChunkHolder implements SortTempChunkHolder {
for (int mesCount = 0; mesCount < measureCount; mesCount++) {
if (UnsafeCarbonRowPage.isSet(words, mesCount)) {
- if (aggType[mesCount] == CarbonCommonConstants.DOUBLE_MEASURE) {
- row[dimensionCount + mesCount] = stream.readDouble();
- } else if (aggType[mesCount] == CarbonCommonConstants.BIG_INT_MEASURE) {
- row[dimensionCount + mesCount] = stream.readLong();
- } else if (aggType[mesCount] == CarbonCommonConstants.BIG_DECIMAL_MEASURE) {
- short aShort = stream.readShort();
- byte[] bigDecimalInBytes = new byte[aShort];
- stream.readFully(bigDecimalInBytes);
- row[dimensionCount + mesCount] = bigDecimalInBytes;
+ switch (measureDataType[mesCount]) {
+ case SHORT:
+ case INT:
+ case LONG:
+ row[dimensionCount + mesCount] = stream.readLong();
+ break;
+ case DOUBLE:
+ row[dimensionCount + mesCount] = stream.readDouble();
+ break;
+ case DECIMAL:
+ short aShort = stream.readShort();
+ byte[] bigDecimalInBytes = new byte[aShort];
+ stream.readFully(bigDecimalInBytes);
+ row[dimensionCount + mesCount] = bigDecimalInBytes;
+ break;
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeIntermediateFileMerger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeIntermediateFileMerger.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeIntermediateFileMerger.java
index e52dc8a..90c3b69 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeIntermediateFileMerger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeIntermediateFileMerger.java
@@ -31,7 +31,7 @@ import java.util.concurrent.Callable;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.processing.newflow.sort.unsafe.UnsafeCarbonRowPage;
import org.apache.carbondata.processing.newflow.sort.unsafe.holder.SortTempChunkHolder;
@@ -278,7 +278,7 @@ public class UnsafeIntermediateFileMerger implements Callable<Void> {
private void writeDataTofile(Object[] row) throws CarbonSortKeyAndGroupByException, IOException {
int dimCount = 0;
int size = 0;
- char[] aggType = mergerParameters.getAggType();
+ DataType[] type = mergerParameters.getMeasureDataType();
for (; dimCount < noDictionarycolumnMapping.length; dimCount++) {
if (noDictionarycolumnMapping[dimCount]) {
byte[] col = (byte[]) row[dimCount];
@@ -310,21 +310,25 @@ public class UnsafeIntermediateFileMerger implements Callable<Void> {
for (int mesCount = 0; mesCount < measureSize; mesCount++) {
Object value = row[mesCount + dimensionSize];
if (null != value) {
- if (aggType[mesCount] == CarbonCommonConstants.DOUBLE_MEASURE) {
- Double val = (Double) value;
- rowData.putDouble(size, val);
- size += 8;
- } else if (aggType[mesCount] == CarbonCommonConstants.BIG_INT_MEASURE) {
- Long val = (Long) value;
- rowData.putLong(size, val);
- size += 8;
- } else if (aggType[mesCount] == CarbonCommonConstants.BIG_DECIMAL_MEASURE) {
- byte[] bigDecimalInBytes = (byte[]) value;
- rowData.putShort(size, (short)bigDecimalInBytes.length);
- size += 2;
- for (int i = 0; i < bigDecimalInBytes.length; i++) {
- rowData.put(size++, bigDecimalInBytes[i]);
- }
+ switch (type[mesCount]) {
+ case SHORT:
+ case INT:
+ case LONG:
+ rowData.putLong(size, (Long) value);
+ size += 8;
+ break;
+ case DOUBLE:
+ rowData.putDouble(size, (Double) value);
+ size += 8;
+ break;
+ case DECIMAL:
+ byte[] bigDecimalInBytes = (byte[]) value;
+ rowData.putShort(size, (short)bigDecimalInBytes.length);
+ size += 2;
+ for (int i = 0; i < bigDecimalInBytes.length; i++) {
+ rowData.put(size++, bigDecimalInBytes[i]);
+ }
+ break;
}
UnsafeCarbonRowPage.set(nullSetWords, mesCount);
} else {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java
index c50f335..0f0a5b0 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java
@@ -26,11 +26,11 @@ import java.util.concurrent.Future;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.keygenerator.KeyGenerator;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.carbondata.processing.newflow.AbstractDataLoadProcessorStep;
@@ -64,7 +64,7 @@ public class CarbonRowDataWriterProcessorStepImpl extends AbstractDataLoadProces
private boolean[] isNoDictionaryDimensionColumn;
- private char[] aggType;
+ private DataType[] measureDataType;
private int dimensionCount;
@@ -115,8 +115,8 @@ public class CarbonRowDataWriterProcessorStepImpl extends AbstractDataLoadProces
dimensionCount = configuration.getDimensionCount() - noDictWithComplextCount;
isNoDictionaryDimensionColumn =
CarbonDataProcessorUtil.getNoDictionaryMapping(configuration.getDataFields());
- aggType = CarbonDataProcessorUtil
- .getAggType(configuration.getMeasureCount(), configuration.getMeasureFields());
+ measureDataType = CarbonDataProcessorUtil
+ .getMeasureDataType(configuration.getMeasureCount(), configuration.getMeasureFields());
CarbonFactDataHandlerModel dataHandlerModel = CarbonFactDataHandlerModel
.createCarbonFactDataHandlerModel(configuration,
@@ -266,7 +266,7 @@ public class CarbonRowDataWriterProcessorStepImpl extends AbstractDataLoadProces
for (; l < this.measureCount; l++) {
Object value = row.getObject(l + this.dimensionWithComplexCount);
if (null != value) {
- if (aggType[l] == CarbonCommonConstants.BIG_DECIMAL_MEASURE) {
+ if (measureDataType[l] == DataType.DECIMAL) {
BigDecimal val = (BigDecimal) value;
outputRow[l] = DataTypeUtil.bigDecimalToByte(val);
} else {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/IntermediateFileMerger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/IntermediateFileMerger.java b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/IntermediateFileMerger.java
index a9e762d..d20292c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/IntermediateFileMerger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/IntermediateFileMerger.java
@@ -29,7 +29,7 @@ import java.util.concurrent.Callable;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.processing.sortandgroupby.exception.CarbonSortKeyAndGroupByException;
import org.apache.carbondata.processing.util.NonDictionaryUtil;
@@ -251,7 +251,8 @@ public class IntermediateFileMerger implements Callable<Void> {
new SortTempFileChunkHolder(tempFile, mergerParameters.getDimColCount(),
mergerParameters.getComplexDimColCount(), mergerParameters.getMeasureColCount(),
mergerParameters.getFileBufferSize(), mergerParameters.getNoDictionaryCount(),
- mergerParameters.getAggType(), mergerParameters.getNoDictionaryDimnesionColumn(),
+ mergerParameters.getMeasureDataType(),
+ mergerParameters.getNoDictionaryDimnesionColumn(),
mergerParameters.getNoDictionarySortColumn());
// initialize
@@ -319,7 +320,7 @@ public class IntermediateFileMerger implements Callable<Void> {
return;
}
try {
- char[] aggType = mergerParameters.getAggType();
+ DataType[] aggType = mergerParameters.getMeasureDataType();
int[] mdkArray = (int[]) row[0];
byte[][] nonDictArray = (byte[][]) row[1];
int mdkIndex = 0;
@@ -339,27 +340,27 @@ public class IntermediateFileMerger implements Callable<Void> {
for (int counter = 0; counter < mergerParameters.getMeasureColCount(); counter++) {
if (null != NonDictionaryUtil.getMeasure(fieldIndex, row)) {
stream.write((byte) 1);
- if (aggType[counter] == CarbonCommonConstants.BYTE_VALUE_MEASURE) {
- Double val = (Double) NonDictionaryUtil.getMeasure(fieldIndex, row);
- stream.writeDouble(val);
- } else if (aggType[counter] == CarbonCommonConstants.DOUBLE_MEASURE) {
- Double val = (Double) NonDictionaryUtil.getMeasure(fieldIndex, row);
- stream.writeDouble(val);
- } else if (aggType[counter] == CarbonCommonConstants.BIG_INT_MEASURE) {
- Long val = (Long) NonDictionaryUtil.getMeasure(fieldIndex, row);
- stream.writeLong(val);
- } else if (aggType[counter] == CarbonCommonConstants.BIG_DECIMAL_MEASURE) {
- byte[] bigDecimalInBytes = (byte[]) NonDictionaryUtil.getMeasure(fieldIndex, row);
- stream.writeInt(bigDecimalInBytes.length);
- stream.write(bigDecimalInBytes);
+ switch (aggType[counter]) {
+ case SHORT:
+ case INT:
+ case LONG:
+ Long val = (Long) NonDictionaryUtil.getMeasure(fieldIndex, row);
+ stream.writeLong(val);
+ break;
+ case DOUBLE:
+ stream.writeDouble((Double) NonDictionaryUtil.getMeasure(fieldIndex, row));
+ break;
+ case DECIMAL:
+ byte[] bigDecimalInBytes = (byte[]) NonDictionaryUtil.getMeasure(fieldIndex, row);
+ stream.writeInt(bigDecimalInBytes.length);
+ stream.write(bigDecimalInBytes);
+ break;
}
} else {
stream.write((byte) 0);
}
-
fieldIndex++;
}
-
} catch (IOException e) {
throw new CarbonSortKeyAndGroupByException("Problem while writing the file", e);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortDataRows.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortDataRows.java b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortDataRows.java
index eba5433..af654e2 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortDataRows.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortDataRows.java
@@ -33,6 +33,7 @@ import java.util.concurrent.TimeUnit;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.DataTypeUtil;
@@ -256,7 +257,7 @@ public class SortDataRows {
stream.writeInt(entryCountLocal);
int complexDimColCount = parameters.getComplexDimColCount();
int dimColCount = parameters.getDimColCount() + complexDimColCount;
- char[] aggType = parameters.getAggType();
+ DataType[] type = parameters.getMeasureDataType();
boolean[] noDictionaryDimnesionMapping = parameters.getNoDictionaryDimnesionColumn();
Object[] row = null;
for (int i = 0; i < entryCountLocal; i++) {
@@ -285,17 +286,21 @@ public class SortDataRows {
Object value = row[mesCount + dimColCount];
if (null != value) {
stream.write((byte) 1);
- if (aggType[mesCount] == CarbonCommonConstants.DOUBLE_MEASURE) {
- Double val = (Double) value;
- stream.writeDouble(val);
- } else if (aggType[mesCount] == CarbonCommonConstants.BIG_INT_MEASURE) {
- Long val = (Long) value;
- stream.writeLong(val);
- } else if (aggType[mesCount] == CarbonCommonConstants.BIG_DECIMAL_MEASURE) {
- BigDecimal val = (BigDecimal) value;
- byte[] bigDecimalInBytes = DataTypeUtil.bigDecimalToByte(val);
- stream.writeInt(bigDecimalInBytes.length);
- stream.write(bigDecimalInBytes);
+ switch (type[mesCount]) {
+ case SHORT:
+ case INT:
+ case LONG:
+ stream.writeLong((Long) value);
+ break;
+ case DOUBLE:
+ stream.writeDouble((Double) value);
+ break;
+ case DECIMAL:
+ BigDecimal val = (BigDecimal) value;
+ byte[] bigDecimalInBytes = DataTypeUtil.bigDecimalToByte(val);
+ stream.writeInt(bigDecimalInBytes.length);
+ stream.write(bigDecimalInBytes);
+ break;
}
} else {
stream.write((byte) 0);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortParameters.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortParameters.java b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortParameters.java
index 7ef8f8e..8ac1491 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortParameters.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortParameters.java
@@ -22,6 +22,7 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.processing.newflow.CarbonDataLoadConfiguration;
import org.apache.carbondata.processing.schema.metadata.SortObserver;
@@ -88,7 +89,7 @@ public class SortParameters {
private String tableName;
- private char[] aggType;
+ private DataType[] measureDataType;
/**
* To know how many columns are of high cardinality.
@@ -137,7 +138,7 @@ public class SortParameters {
parameters.bufferSize = bufferSize;
parameters.databaseName = databaseName;
parameters.tableName = tableName;
- parameters.aggType = aggType;
+ parameters.measureDataType = measureDataType;
parameters.noDictionaryCount = noDictionaryCount;
parameters.partitionID = partitionID;
parameters.segmentId = segmentId;
@@ -270,12 +271,12 @@ public class SortParameters {
this.tableName = tableName;
}
- public char[] getAggType() {
- return aggType;
+ public DataType[] getMeasureDataType() {
+ return measureDataType;
}
- public void setAggType(char[] aggType) {
- this.aggType = aggType;
+ public void setMeasureDataType(DataType[] measureDataType) {
+ this.measureDataType = measureDataType;
}
public int getNoDictionaryCount() {
@@ -458,9 +459,9 @@ public class SortParameters {
CarbonCommonConstants.CARBON_PREFETCH_BUFFERSIZE,
CarbonCommonConstants.CARBON_PREFETCH_BUFFERSIZE_DEFAULT)));
- char[] aggType = CarbonDataProcessorUtil
- .getAggType(configuration.getMeasureCount(), configuration.getMeasureFields());
- parameters.setAggType(aggType);
+ DataType[] measureDataType = CarbonDataProcessorUtil
+ .getMeasureDataType(configuration.getMeasureCount(), configuration.getMeasureFields());
+ parameters.setMeasureDataType(measureDataType);
return parameters;
}
@@ -560,10 +561,10 @@ public class SortParameters {
CarbonCommonConstants.CARBON_PREFETCH_BUFFERSIZE,
CarbonCommonConstants.CARBON_PREFETCH_BUFFERSIZE_DEFAULT)));
- char[] aggType = CarbonDataProcessorUtil
- .getAggType(parameters.getMeasureColCount(), parameters.getDatabaseName(),
+ DataType[] type = CarbonDataProcessorUtil
+ .getMeasureDataType(parameters.getMeasureColCount(), parameters.getDatabaseName(),
parameters.getTableName());
- parameters.setAggType(aggType);
+ parameters.setMeasureDataType(type);
return parameters;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/98df130a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortTempFileChunkHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortTempFileChunkHolder.java b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortTempFileChunkHolder.java
index 6695a5b..a4fdec1 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortTempFileChunkHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortTempFileChunkHolder.java
@@ -31,6 +31,7 @@ import java.util.concurrent.Future;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.util.ByteUtil.UnsafeComparer;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
@@ -125,7 +126,7 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
private int noDictionaryCount;
- private char[] aggType;
+ private DataType[] aggType;
/**
* to store whether dimension is of dictionary type or not
@@ -150,7 +151,7 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
* @param isNoDictionaryDimensionColumn
*/
public SortTempFileChunkHolder(File tempFile, int dimensionCount, int complexDimensionCount,
- int measureCount, int fileBufferSize, int noDictionaryCount, char[] aggType,
+ int measureCount, int fileBufferSize, int noDictionaryCount, DataType[] aggType,
boolean[] isNoDictionaryDimensionColumn, boolean[] isNoDictionarySortColumn) {
// set temp file
this.tempFile = tempFile;
@@ -338,15 +339,21 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
// read measure values
for (int i = 0; i < this.measureCount; i++) {
if (stream.readByte() == 1) {
- if (aggType[i] == CarbonCommonConstants.DOUBLE_MEASURE) {
- measures[index++] = stream.readDouble();
- } else if (aggType[i] == CarbonCommonConstants.BIG_INT_MEASURE) {
- measures[index++] = stream.readLong();
- } else {
- int len = stream.readInt();
- byte[] buff = new byte[len];
- stream.readFully(buff);
- measures[index++] = buff;
+ switch (aggType[i]) {
+ case SHORT:
+ case INT:
+ case LONG:
+ measures[index++] = stream.readLong();
+ break;
+ case DOUBLE:
+ measures[index++] = stream.readDouble();
+ break;
+ case DECIMAL:
+ int len = stream.readInt();
+ byte[] buff = new byte[len];
+ stream.readFully(buff);
+ measures[index++] = buff;
+ break;
}
} else {
measures[index++] = null;
[04/19] carbondata git commit: [CARBONDATA-886] remove redundant
variable
Posted by ra...@apache.org.
[CARBONDATA-886] remove redundant variable
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/81149f65
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/81149f65
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/81149f65
Branch: refs/heads/master
Commit: 81149f659cee4fe1baa1192d8e3b757a48b5d16d
Parents: 4827315
Author: QiangCai <qi...@qq.com>
Authored: Tue May 2 09:43:23 2017 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Sun May 14 20:40:11 2017 +0800
----------------------------------------------------------------------
.../dictionary/DictionaryCacheLoaderImpl.java | 3 +--
.../dictionary/DoubleArrayTrieDictionary.java | 5 ++--
.../dictionary/ReverseDictionaryCache.java | 3 +--
.../block/TableBlockUniqueIdentifier.java | 3 +--
.../core/datastore/impl/FileFactory.java | 10 +++-----
.../DateDirectDictionaryGenerator.java | 3 +--
.../core/keygenerator/mdkey/Bits.java | 3 +--
.../keygenerator/mdkey/NumberCompressor.java | 7 +++--
.../core/mutate/CarbonUpdateUtil.java | 14 ++++------
.../CarbonDictionaryMetadataReaderImpl.java | 12 +++------
.../core/reader/CarbonIndexFileReader.java | 6 ++---
.../core/scan/expression/ColumnExpression.java | 4 +--
.../core/scan/expression/LiteralExpression.java | 6 ++---
.../executer/RowLevelFilterExecuterImpl.java | 6 ++---
.../executer/ValueBasedFilterExecuterImpl.java | 6 ++---
.../iterator/DetailQueryResultIterator.java | 3 +--
.../SegmentUpdateStatusManager.java | 5 ++--
.../apache/carbondata/core/util/CarbonUtil.java | 15 +++++------
.../core/util/path/CarbonStorePath.java | 3 +--
.../core/util/path/CarbonTablePath.java | 6 ++---
.../dictionary/AbstractDictionaryCacheTest.java | 6 ++---
.../dictionary/ReverseDictionaryCacheTest.java | 4 +--
.../block/SegmentPropertiesTestUtil.java | 3 +--
.../filesystem/AlluxioCarbonFileTest.java | 6 ++---
.../filesystem/HDFSCarbonFileTest.java | 9 +++----
.../filesystem/LocalCarbonFileTest.java | 3 +--
.../filesystem/ViewFsCarbonFileTest.java | 6 ++---
.../ThriftWrapperSchemaConverterImplTest.java | 3 +--
.../core/scan/executor/util/QueryUtilTest.java | 9 +++----
.../core/util/RangeFilterProcessorTest.java | 21 +++++----------
.../processing/merger/CarbonCompactionUtil.java | 3 +--
.../processing/merger/CarbonDataMergerUtil.java | 27 ++++++++------------
.../merger/CompactionResultSortProcessor.java | 3 +--
.../newflow/DataLoadProcessBuilder.java | 12 +++------
.../DictionaryServerClientDictionary.java | 3 +--
...arallelReadMergeSorterWithBucketingImpl.java | 4 +--
.../holder/UnsafeSortTempFileChunkHolder.java | 3 +--
.../steps/DataConverterProcessorStepImpl.java | 3 +--
...ConverterProcessorWithBucketingStepImpl.java | 3 +--
.../newflow/steps/SortProcessorStepImpl.java | 3 +--
.../sortdata/SortTempFileChunkHolder.java | 3 +--
.../store/CarbonFactDataHandlerModel.java | 14 ++++------
.../util/CarbonDataProcessorUtil.java | 3 +--
.../processing/util/NonDictionaryUtil.java | 3 +--
44 files changed, 94 insertions(+), 186 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/81149f65/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java
index 4a0c2e8..5373ca8 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java
@@ -128,8 +128,7 @@ public class DictionaryCacheLoaderImpl implements DictionaryCacheLoader {
throws IOException {
CarbonDictionaryReader dictionaryReader = getDictionaryReader(columnIdentifier);
try {
- Iterator<byte[]> columnDictionaryChunkWrapper = dictionaryReader.read(startOffset, endOffset);
- return columnDictionaryChunkWrapper;
+ return dictionaryReader.read(startOffset, endOffset);
} finally {
dictionaryReader.close();
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/81149f65/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java
index 26b12bf..aeec846 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java
@@ -223,11 +223,10 @@ public class DoubleArrayTrieDictionary {
*/
private int conflict(int start, int bKey) {
int from = start;
- int newKey = bKey;
TreeSet<Integer> children = getChildren(from);
- children.add(new Integer(newKey));
+ children.add(new Integer(bKey));
int newBasePos = findFreeRoom(children);
- children.remove(new Integer(newKey));
+ children.remove(new Integer(bKey));
int oldBasePos = base[start];
base[start] = newBasePos;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/81149f65/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCache.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCache.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCache.java
index 55dfa0e..acb9acf 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCache.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCache.java
@@ -89,8 +89,7 @@ public class ReverseDictionaryCache<K extends DictionaryColumnUniqueIdentifier,
for (final DictionaryColumnUniqueIdentifier uniqueIdent : dictionaryColumnUniqueIdentifiers) {
taskSubmitList.add(executorService.submit(new Callable<Dictionary>() {
@Override public Dictionary call() throws IOException {
- Dictionary dictionary = getDictionary(uniqueIdent);
- return dictionary;
+ return getDictionary(uniqueIdent);
}
}));
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/81149f65/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockUniqueIdentifier.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockUniqueIdentifier.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockUniqueIdentifier.java
index e873b85..0434066 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockUniqueIdentifier.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockUniqueIdentifier.java
@@ -78,11 +78,10 @@ public class TableBlockUniqueIdentifier {
BlockInfo blockInfo = new BlockInfo(this.tableBlockInfo);
CarbonTableIdentifier carbonTableIdentifier =
this.absoluteTableIdentifier.getCarbonTableIdentifier();
- String uniqueTableBlockName = carbonTableIdentifier.getDatabaseName()
+ return carbonTableIdentifier.getDatabaseName()
+ CarbonCommonConstants.FILE_SEPARATOR + carbonTableIdentifier
.getDatabaseName() + CarbonCommonConstants.FILE_SEPARATOR
+ this.tableBlockInfo.getSegmentId()
+ CarbonCommonConstants.FILE_SEPARATOR + blockInfo.hashCode();
- return uniqueTableBlockName;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/81149f65/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
index c32b956..ae7faf8 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
@@ -191,8 +191,7 @@ public final class FileFactory {
case VIEWFS:
Path pt = new Path(path);
FileSystem fs = pt.getFileSystem(configuration);
- FSDataOutputStream stream = fs.create(pt, true);
- return stream;
+ return fs.create(pt, true);
default:
return new DataOutputStream(new BufferedOutputStream(new FileOutputStream(path)));
}
@@ -244,9 +243,7 @@ public final class FileFactory {
case VIEWFS:
Path pt = new Path(path);
FileSystem fs = pt.getFileSystem(configuration);
- FSDataOutputStream stream =
- fs.create(pt, true, bufferSize, fs.getDefaultReplication(pt), blockSize);
- return stream;
+ return fs.create(pt, true, bufferSize, fs.getDefaultReplication(pt), blockSize);
default:
path = getUpdatedFilePath(path, fileType);
return new DataOutputStream(
@@ -402,8 +399,7 @@ public final class FileFactory {
case VIEWFS:
Path pt = new Path(path);
FileSystem fs = pt.getFileSystem(configuration);
- FSDataOutputStream stream = fs.append(pt);
- return stream;
+ return fs.append(pt);
default:
return new DataOutputStream(new BufferedOutputStream(new FileOutputStream(path)));
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/81149f65/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
index 8db42b0..a21fd85 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
@@ -152,8 +152,7 @@ public class DateDirectDictionaryGenerator implements DirectDictionaryGenerator
private int generateKey(long timeValue) {
long milli = timeValue + threadLocalLocalTimeZone.get().getOffset(timeValue);
- int key = (int) Math.floor((double) milli / MILLIS_PER_DAY) + cutOffDate;
- return key;
+ return (int) Math.floor((double) milli / MILLIS_PER_DAY) + cutOffDate;
}
public void initialize() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/81149f65/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/Bits.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/Bits.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/Bits.java
index 044a77b..822b343 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/Bits.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/Bits.java
@@ -89,9 +89,8 @@ public class Bits implements Serializable {
* {start, end}
*/
public int[] getKeyByteOffsets(int index) {
- int prefixPaddingBits = length % 8 == 0 ? 0 : (8 - length % 8);
- int priorLen = prefixPaddingBits;
+ int priorLen = length % 8 == 0 ? 0 : (8 - length % 8);
int start = 0;
int end = 0;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/81149f65/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/NumberCompressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/NumberCompressor.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/NumberCompressor.java
index af58f50..d601159 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/NumberCompressor.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/NumberCompressor.java
@@ -138,10 +138,9 @@ public class NumberCompressor {
}
public int[] unCompress(byte[] key, int offset, int length) {
- int ls = length;
- int arrayLength = (ls * BYTE_LENGTH) / bitsLength;
- long[] words = new long[getWordsSizeFromBytesSize(ls)];
- unCompressVal(key, ls, words, offset);
+ int arrayLength = (length * BYTE_LENGTH) / bitsLength;
+ long[] words = new long[getWordsSizeFromBytesSize(length)];
+ unCompressVal(key, length, words, offset);
return getArray(words, arrayLength);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/81149f65/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
index fef5905..af26035 100644
--- a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
@@ -340,8 +340,7 @@ public class CarbonUpdateUtil {
*/
public static Long getTimeStampAsLong(String timtstamp) {
try {
- long longValue = Long.parseLong(timtstamp);
- return longValue;
+ return Long.parseLong(timtstamp);
} catch (NumberFormatException nfe) {
String errorMsg = "Invalid timestamp : " + timtstamp;
LOGGER.error(errorMsg);
@@ -358,8 +357,7 @@ public class CarbonUpdateUtil {
*/
public static Integer getIntegerValue(String value) throws Exception {
try {
- int intValue = Integer.parseInt(value);
- return intValue;
+ return Integer.parseInt(value);
} catch (NumberFormatException nfe) {
LOGGER.error("Invalid row : " + value + nfe.getLocalizedMessage());
throw new Exception("Invalid row : " + nfe.getLocalizedMessage());
@@ -373,9 +371,8 @@ public class CarbonUpdateUtil {
* @return
*/
public static String getBlockName(String completeBlockName) {
- String blockName =
- completeBlockName.substring(0, completeBlockName.lastIndexOf(CarbonCommonConstants.HYPHEN));
- return blockName;
+ return completeBlockName
+ .substring(0, completeBlockName.lastIndexOf(CarbonCommonConstants.HYPHEN));
}
/**
@@ -385,8 +382,7 @@ public class CarbonUpdateUtil {
* @return
*/
public static String getSegmentId(String segmentName) {
- String id = segmentName.split(CarbonCommonConstants.UNDERSCORE)[1];
- return id;
+ return segmentName.split(CarbonCommonConstants.UNDERSCORE)[1];
}
public static int getLatestTaskIdForSegment(String segmentId, CarbonTablePath tablePath) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/81149f65/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java
index 79ca689..98c0ddb 100644
--- a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java
@@ -130,9 +130,7 @@ public class CarbonDictionaryMetadataReaderImpl implements CarbonDictionaryMetad
dictionaryChunkMeta = (ColumnDictionaryChunkMeta) dictionaryMetadataFileReader.read();
}
// create a new instance of chunk meta wrapper using thrift object
- CarbonDictionaryColumnMetaChunk columnMetaChunkForLastSegment =
- getNewInstanceOfCarbonDictionaryColumnMetaChunk(dictionaryChunkMeta);
- return columnMetaChunkForLastSegment;
+ return getNewInstanceOfCarbonDictionaryColumnMetaChunk(dictionaryChunkMeta);
}
/**
@@ -190,10 +188,8 @@ public class CarbonDictionaryMetadataReaderImpl implements CarbonDictionaryMetad
*/
private CarbonDictionaryColumnMetaChunk getNewInstanceOfCarbonDictionaryColumnMetaChunk(
ColumnDictionaryChunkMeta dictionaryChunkMeta) {
- CarbonDictionaryColumnMetaChunk columnMetaChunk =
- new CarbonDictionaryColumnMetaChunk(dictionaryChunkMeta.getMin_surrogate_key(),
- dictionaryChunkMeta.getMax_surrogate_key(), dictionaryChunkMeta.getStart_offset(),
- dictionaryChunkMeta.getEnd_offset(), dictionaryChunkMeta.getChunk_count());
- return columnMetaChunk;
+ return new CarbonDictionaryColumnMetaChunk(dictionaryChunkMeta.getMin_surrogate_key(),
+ dictionaryChunkMeta.getMax_surrogate_key(), dictionaryChunkMeta.getStart_offset(),
+ dictionaryChunkMeta.getEnd_offset(), dictionaryChunkMeta.getChunk_count());
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/81149f65/core/src/main/java/org/apache/carbondata/core/reader/CarbonIndexFileReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/reader/CarbonIndexFileReader.java b/core/src/main/java/org/apache/carbondata/core/reader/CarbonIndexFileReader.java
index b2c2cb5..dc46242 100644
--- a/core/src/main/java/org/apache/carbondata/core/reader/CarbonIndexFileReader.java
+++ b/core/src/main/java/org/apache/carbondata/core/reader/CarbonIndexFileReader.java
@@ -40,12 +40,11 @@ public class CarbonIndexFileReader {
* @throws IOException if any problem while reader the header
*/
public IndexHeader readIndexHeader() throws IOException {
- IndexHeader indexHeader = (IndexHeader) thriftReader.read(new ThriftReader.TBaseCreator() {
+ return (IndexHeader) thriftReader.read(new ThriftReader.TBaseCreator() {
@Override public TBase create() {
return new IndexHeader();
}
});
- return indexHeader;
}
/**
@@ -62,12 +61,11 @@ public class CarbonIndexFileReader {
* @throws IOException if problem while reading the block index
*/
public BlockIndex readBlockIndexInfo() throws IOException {
- BlockIndex blockInfo = (BlockIndex) thriftReader.read(new ThriftReader.TBaseCreator() {
+ return (BlockIndex) thriftReader.read(new ThriftReader.TBaseCreator() {
@Override public TBase create() {
return new BlockIndex();
}
});
- return blockInfo;
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/81149f65/core/src/main/java/org/apache/carbondata/core/scan/expression/ColumnExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/ColumnExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/ColumnExpression.java
index 69fa9d6..d27d789 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/ColumnExpression.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/ColumnExpression.java
@@ -86,9 +86,7 @@ public class ColumnExpression extends LeafExpression {
}
@Override public ExpressionResult evaluate(RowIntf value) {
- ExpressionResult expressionResult =
- new ExpressionResult(dataType, (null == value ? null : value.getVal(colIndex)));
- return expressionResult;
+ return new ExpressionResult(dataType, (null == value ? null : value.getVal(colIndex)));
}
@Override public ExpressionType getFilterExpressionType() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/81149f65/core/src/main/java/org/apache/carbondata/core/scan/expression/LiteralExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/LiteralExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/LiteralExpression.java
index 112bf03..b194e3c 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/LiteralExpression.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/LiteralExpression.java
@@ -37,13 +37,11 @@ public class LiteralExpression extends LeafExpression {
}
@Override public ExpressionResult evaluate(RowIntf value) {
- ExpressionResult expressionResult = new ExpressionResult(dataType, this.value, true);
- return expressionResult;
+ return new ExpressionResult(dataType, this.value, true);
}
public ExpressionResult getExpressionResult() {
- ExpressionResult expressionResult = new ExpressionResult(dataType, this.value, true);
- return expressionResult;
+ return new ExpressionResult(dataType, this.value, true);
}
@Override public ExpressionType getFilterExpressionType() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/81149f65/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
index 470de89..8e7c95f 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
@@ -430,8 +430,7 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
if (dimColumnEvaluatorInfo.getDimension().isColumnar()) {
byte[] rawData = dataChunk.getChunkData(index);
ByteBuffer byteBuffer = ByteBuffer.allocate(CarbonCommonConstants.INT_SIZE_IN_BYTE);
- int dictionaryValue = CarbonUtil.getSurrogateKey(rawData, byteBuffer);
- return dictionaryValue;
+ return CarbonUtil.getSurrogateKey(rawData, byteBuffer);
} else {
return readSurrogatesFromColumnGroupBlock(dataChunk, index, dimColumnEvaluatorInfo);
}
@@ -452,9 +451,8 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
long[] result = keyStructureInfo.getKeyGenerator().getKeyArray(colData);
int colGroupId =
QueryUtil.getColumnGroupId(segmentProperties, dimensionBlocksIndex[0]);
- int dictionaryValue = (int) result[segmentProperties
+ return (int) result[segmentProperties
.getColumnGroupMdKeyOrdinal(colGroupId, dimensionBlocksIndex[0])];
- return dictionaryValue;
} catch (KeyGenException e) {
LOGGER.error(e);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/81149f65/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java
index 003d15e..3e33d04 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java
@@ -28,13 +28,11 @@ public class ValueBasedFilterExecuterImpl implements FilterExecuter {
@Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder)
throws FilterUnsupportedException, IOException {
- BitSetGroup bitSetGroup = new BitSetGroup(0);
- return bitSetGroup;
+ return new BitSetGroup(0);
}
@Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
- BitSet bitSet = new BitSet(1);
- return bitSet;
+ return new BitSet(1);
}
@Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/81149f65/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/DetailQueryResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/DetailQueryResultIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/DetailQueryResultIterator.java
index e39b93f..747f5a9 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/DetailQueryResultIterator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/DetailQueryResultIterator.java
@@ -38,8 +38,7 @@ public class DetailQueryResultIterator extends AbstractDetailQueryResultIterator
}
@Override public BatchResult next() {
- BatchResult batchResult = getBatchResult();
- return batchResult;
+ return getBatchResult();
}
private BatchResult getBatchResult() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/81149f65/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
index c822935..70e1993 100644
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
@@ -427,7 +427,8 @@ public class SegmentUpdateStatusManager {
getStartTimeOfDeltaFile(CarbonCommonConstants.DELETE_DELTA_FILE_EXT, block);
final long deltaEndTimeStamp =
getEndTimeOfDeltaFile(CarbonCommonConstants.DELETE_DELTA_FILE_EXT, block);
- CarbonFile[] files = segDir.listFiles(new CarbonFileFilter() {
+
+ return segDir.listFiles(new CarbonFileFilter() {
@Override public boolean accept(CarbonFile pathName) {
String fileName = pathName.getName();
@@ -444,8 +445,6 @@ public class SegmentUpdateStatusManager {
return false;
}
});
-
- return files;
}
}
return null;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/81149f65/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 95ee94b..99463de 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -722,9 +722,8 @@ public final class CarbonUtil {
if (null == prop) {
return null;
}
- String basePath = prop.getProperty(CarbonCommonConstants.STORE_LOCATION,
+ return prop.getProperty(CarbonCommonConstants.STORE_LOCATION,
CarbonCommonConstants.STORE_LOCATION_DEFAULT_VAL);
- return basePath;
}
/**
@@ -1057,9 +1056,8 @@ public final class CarbonUtil {
isDictionaryDimensions.add(false);
}
}
- boolean[] primitive = ArrayUtils
+ return ArrayUtils
.toPrimitive(isDictionaryDimensions.toArray(new Boolean[isDictionaryDimensions.size()]));
- return primitive;
}
/**
@@ -1594,14 +1592,13 @@ public final class CarbonUtil {
return true;
}
- UpdateVO updatedVODetails = invalidBlockVOForSegmentId;
- if (null != updatedVODetails) {
+ if (null != invalidBlockVOForSegmentId) {
Long blockTimeStamp = Long.parseLong(tableInfo.getFilePath()
.substring(tableInfo.getFilePath().lastIndexOf('-') + 1,
tableInfo.getFilePath().lastIndexOf('.')));
- if ((blockTimeStamp > updatedVODetails.getFactTimestamp() && (
- updatedVODetails.getUpdateDeltaStartTimestamp() != null
- && blockTimeStamp < updatedVODetails.getUpdateDeltaStartTimestamp()))) {
+ if ((blockTimeStamp > invalidBlockVOForSegmentId.getFactTimestamp() && (
+ invalidBlockVOForSegmentId.getUpdateDeltaStartTimestamp() != null
+ && blockTimeStamp < invalidBlockVOForSegmentId.getUpdateDeltaStartTimestamp()))) {
return true;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/81149f65/core/src/main/java/org/apache/carbondata/core/util/path/CarbonStorePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonStorePath.java b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonStorePath.java
index 54de175..3aece69 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonStorePath.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonStorePath.java
@@ -40,10 +40,9 @@ public class CarbonStorePath extends Path {
*/
public static CarbonTablePath getCarbonTablePath(String storePath,
CarbonTableIdentifier tableIdentifier) {
- CarbonTablePath carbonTablePath = new CarbonTablePath(tableIdentifier,
+ return new CarbonTablePath(tableIdentifier,
storePath + File.separator + tableIdentifier.getDatabaseName() + File.separator
+ tableIdentifier.getTableName());
- return carbonTablePath;
}
public static CarbonTablePath getCarbonTablePath(String storePath,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/81149f65/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
index 918772a..6a60a65 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
@@ -590,12 +590,11 @@ public class CarbonTablePath extends Path {
* @return sort index carbon files
*/
public CarbonFile[] getSortIndexFiles(CarbonFile sortIndexDir, final String columnUniqueId) {
- CarbonFile[] files = sortIndexDir.listFiles(new CarbonFileFilter() {
+ return sortIndexDir.listFiles(new CarbonFileFilter() {
@Override public boolean accept(CarbonFile file) {
return file.getName().startsWith(columnUniqueId) && file.getName().endsWith(SORT_INDEX_EXT);
}
});
- return files;
}
/**
@@ -605,10 +604,9 @@ public class CarbonTablePath extends Path {
* @return
*/
public static String getCarbonDataFileName(String carbonDataFilePath) {
- String carbonDataFileName = carbonDataFilePath
+ return carbonDataFilePath
.substring(carbonDataFilePath.lastIndexOf(CarbonCommonConstants.FILE_SEPARATOR) + 1,
carbonDataFilePath.indexOf(CARBON_DATA_EXT));
- return carbonDataFileName;
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/81149f65/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java
index 2bbe458..2fa2b93 100644
--- a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java
@@ -104,10 +104,8 @@ public class AbstractDictionaryCacheTest {
protected DictionaryColumnUniqueIdentifier createDictionaryColumnUniqueIdentifier(
String columnId) {
ColumnIdentifier columnIdentifier = new ColumnIdentifier(columnId, null, DataType.STRING);
- DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier =
- new DictionaryColumnUniqueIdentifier(carbonTableIdentifier, columnIdentifier,
- DataType.STRING);
- return dictionaryColumnUniqueIdentifier;
+ return new DictionaryColumnUniqueIdentifier(carbonTableIdentifier, columnIdentifier,
+ DataType.STRING);
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/81149f65/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCacheTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCacheTest.java b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCacheTest.java
index f5eff67..628c3ff 100644
--- a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCacheTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCacheTest.java
@@ -271,8 +271,6 @@ public class ReverseDictionaryCacheTest extends AbstractDictionaryCacheTest {
protected DictionaryColumnUniqueIdentifier createDictionaryColumnUniqueIdentifier(
String columnId) {
ColumnIdentifier columnIdentifier = new ColumnIdentifier(columnId, null, DataType.DOUBLE);
- DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier =
- new DictionaryColumnUniqueIdentifier(carbonTableIdentifier, columnIdentifier);
- return dictionaryColumnUniqueIdentifier;
+ return new DictionaryColumnUniqueIdentifier(carbonTableIdentifier, columnIdentifier);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/81149f65/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTestUtil.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTestUtil.java b/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTestUtil.java
index 2bfe40d..5a941b3 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTestUtil.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTestUtil.java
@@ -47,8 +47,7 @@ public class SegmentPropertiesTestUtil {
cardinality[i] = x;
x++;
}
- SegmentProperties segmentProperties = new SegmentProperties(columnSchema, cardinality);
- return segmentProperties;
+ return new SegmentProperties(columnSchema, cardinality);
}
public static ColumnSchema getDimensionColumn1() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/81149f65/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFileTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFileTest.java
index a691dc7..eff0d8b 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFileTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFileTest.java
@@ -135,8 +135,7 @@ public class AlluxioCarbonFileTest {
@Mock
public FileStatus[] listStatus(Path var1) throws IOException {
- FileStatus[] fileStatus = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
- return fileStatus;
+ return new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
}
};
@@ -207,8 +206,7 @@ public class AlluxioCarbonFileTest {
@Mock
public FileStatus[] listStatus(Path var1) throws IOException {
- FileStatus[] fileStatus = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
- return fileStatus;
+ return new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
}
};
http://git-wip-us.apache.org/repos/asf/carbondata/blob/81149f65/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java
index b9c1802..5ca7fa8 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java
@@ -176,8 +176,7 @@ public class HDFSCarbonFileTest {
@Mock
public FileStatus[] listStatus(Path var1) throws IOException {
- FileStatus[] fileStatus = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
- return fileStatus;
+ return new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
}
};
@@ -251,8 +250,7 @@ public class HDFSCarbonFileTest {
@Mock
public FileStatus[] listStatus(Path var1) throws IOException {
- FileStatus fileStatus[] = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
- return fileStatus;
+ return new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
}
};
@@ -280,8 +278,7 @@ public class HDFSCarbonFileTest {
@Mock
public FileStatus[] listStatus(Path var1) throws IOException {
- FileStatus[] fileStatus = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
- return fileStatus;
+ return new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
}
};
http://git-wip-us.apache.org/repos/asf/carbondata/blob/81149f65/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFileTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFileTest.java
index 498f4fc..3925e9e 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFileTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFileTest.java
@@ -294,8 +294,7 @@ public class LocalCarbonFileTest {
@Mock
public File[] listFiles(FileFilter filter) {
- File[] file = new File[]{dir};
- return file;
+ return new File[]{dir};
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/81149f65/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/ViewFsCarbonFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/ViewFsCarbonFileTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/ViewFsCarbonFileTest.java
index 5218043..3cb8198 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/ViewFsCarbonFileTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/ViewFsCarbonFileTest.java
@@ -145,8 +145,7 @@ public class ViewFsCarbonFileTest {
@Mock
public FileStatus[] listStatus(Path var1) throws IOException {
- FileStatus[] fileStatus = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
- return fileStatus;
+ return new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
}
};
@@ -217,8 +216,7 @@ public class ViewFsCarbonFileTest {
@Mock
public FileStatus[] listStatus(Path var1) throws IOException {
- FileStatus[] fileStatus = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
- return fileStatus;
+ return new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
}
};
http://git-wip-us.apache.org/repos/asf/carbondata/blob/81149f65/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java b/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
index b044b6f..e8851e0 100644
--- a/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
@@ -105,8 +105,7 @@ public class ThriftWrapperSchemaConverterImplTest {
}
@Mock public long getTime_stamp() {
- long time = 1112745600000L;
- return time;
+ return 1112745600000L;
}
@Mock public List<org.apache.carbondata.format.ColumnSchema> getAdded() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/81149f65/core/src/test/java/org/apache/carbondata/core/scan/executor/util/QueryUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/executor/util/QueryUtilTest.java b/core/src/test/java/org/apache/carbondata/core/scan/executor/util/QueryUtilTest.java
index 288c148..23504d3 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/executor/util/QueryUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/executor/util/QueryUtilTest.java
@@ -149,8 +149,7 @@ public class QueryUtilTest extends TestCase {
int ordinal = 0;
new MockUp<SegmentProperties>() {
@Mock public int[][] getColumnGroups() {
- int columnGroups[][] = { { 1, 1 }, { 2, 2 }, { 3, 3 }, { 4, 4 }, { 5, 5 } };
- return columnGroups;
+ return new int[][] { { 1, 1 }, { 2, 2 }, { 3, 3 }, { 4, 4 }, { 5, 5 } };
}
};
int actualValue = QueryUtil.getColumnGroupId(segmentProperties, ordinal);
@@ -162,8 +161,7 @@ public class QueryUtilTest extends TestCase {
int ordinal = 1;
new MockUp<SegmentProperties>() {
@Mock public int[][] getColumnGroups() {
- int columnGroups[][] = { { 1, 1 }, { 2, 2 }, { 3, 3 }, { 4, 4 }, { 5, 5 } };
- return columnGroups;
+ return new int[][] { { 1, 1 }, { 2, 2 }, { 3, 3 }, { 4, 4 }, { 5, 5 } };
}
};
int actualValue = QueryUtil.getColumnGroupId(segmentProperties, ordinal);
@@ -175,8 +173,7 @@ public class QueryUtilTest extends TestCase {
int ordinal = 1;
new MockUp<SegmentProperties>() {
@Mock public int[][] getColumnGroups() {
- int columnGroups[][] = { { 1 } };
- return columnGroups;
+ return new int[][] { { 1 } };
}
};
int actualValue = QueryUtil.getColumnGroupId(segmentProperties, ordinal);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/81149f65/core/src/test/java/org/apache/carbondata/core/util/RangeFilterProcessorTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/RangeFilterProcessorTest.java b/core/src/test/java/org/apache/carbondata/core/util/RangeFilterProcessorTest.java
index 2d8927b..952b7f7 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/RangeFilterProcessorTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/RangeFilterProcessorTest.java
@@ -94,8 +94,7 @@ public class RangeFilterProcessorTest {
colb.setDimension(empDimension);
Expression lessThan =
new LessThanEqualToExpression(colb, new LiteralExpression("20", DataType.STRING));
- Expression andExp = new AndExpression(greaterThan, lessThan);
- inputFilter = andExp;
+ inputFilter = new AndExpression(greaterThan, lessThan);
Expression output = new AndExpression(new RangeExpression(
new GreaterThanEqualToExpression(new ColumnExpression("a", DataType.STRING),
@@ -136,8 +135,7 @@ public class RangeFilterProcessorTest {
colb.setDimension(empDimension);
Expression lessThan =
new LessThanEqualToExpression(colb, new LiteralExpression("05", DataType.STRING));
- Expression andExp = new AndExpression(greaterThan, lessThan);
- inputFilter = andExp;
+ inputFilter = new AndExpression(greaterThan, lessThan);
Expression output = new AndExpression(
new GreaterThanEqualToExpression(new ColumnExpression("a", DataType.STRING),
@@ -195,8 +193,7 @@ public class RangeFilterProcessorTest {
Expression And1 = new AndExpression(new NotEqualsExpression(null, null), greaterThan2);
Expression And2 = new AndExpression(And1, greaterThan1);
Expression And3 = new AndExpression(And2, lessThan2);
- Expression And4 = new AndExpression(And3, lessThan1);
- inputFilter = And4;
+ inputFilter = new AndExpression(And3, lessThan1);
// Build The output
@@ -219,14 +216,11 @@ public class RangeFilterProcessorTest {
Expression Andb2 = new AndExpression(Andb1, new RangeExpression(greaterThanb1, lessThanb1));
Expression Andb3 = new AndExpression(Andb2, new TrueExpression(null));
- Expression Andb4 = new AndExpression(Andb3, new TrueExpression(null));
-
- Expression output = Andb4;
FilterOptimizer rangeFilterOptimizer =
new RangeFilterOptmizer(new FilterOptimizerBasic(), inputFilter);
rangeFilterOptimizer.optimizeFilter();
- result = checkBothTrees(inputFilter, output);
+ result = checkBothTrees(inputFilter, new AndExpression(Andb3, new TrueExpression(null)));
// no change
Assert.assertTrue(result);
}
@@ -274,8 +268,7 @@ public class RangeFilterProcessorTest {
Expression Or1 = new OrExpression(new NotEqualsExpression(null, null), greaterThan2);
Expression Or2 = new OrExpression(Or1, greaterThan1);
Expression Or3 = new OrExpression(Or2, lessThan2);
- Expression Or4 = new OrExpression(Or3, lessThan1);
- inputFilter = Or4;
+ inputFilter = new OrExpression(Or3, lessThan1);
// Build The output
@@ -307,13 +300,11 @@ public class RangeFilterProcessorTest {
Expression Orb1 = new OrExpression(new NotEqualsExpression(null, null), greaterThanb2);
Expression Orb2 = new OrExpression(Orb1, greaterThanb1);
Expression Orb3 = new OrExpression(Orb2, lessThanb2);
- Expression Orb4 = new OrExpression(Orb3, lessThanb1);
- Expression output = Orb4;
FilterOptimizer rangeFilterOptimizer =
new RangeFilterOptmizer(new FilterOptimizerBasic(), inputFilter);
rangeFilterOptimizer.optimizeFilter();
- result = checkBothTrees(inputFilter, output);
+ result = checkBothTrees(inputFilter, new OrExpression(Orb3, lessThanb1));
// no change
Assert.assertTrue(result);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/81149f65/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java
index 2ad83a4..b3d5b5c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java
@@ -325,9 +325,8 @@ public class CarbonCompactionUtil {
for (CarbonMeasure measure : masterSchemaMeasures) {
updatedColumnSchemaList.add(measure.getColumnSchema());
}
- int[] updatedCardinality = ArrayUtils
+ return ArrayUtils
.toPrimitive(updatedCardinalityList.toArray(new Integer[updatedCardinalityList.size()]));
- return updatedCardinality;
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/81149f65/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
index b9ea447..ce9c433 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
@@ -139,16 +139,13 @@ public final class CarbonDataMergerUtil {
* @return
*/
public static String getMergedLoadName(final String segmentToBeMerged) {
- String firstSegmentName = segmentToBeMerged;
- if (firstSegmentName.contains(".")) {
- String beforeDecimal = firstSegmentName.substring(0, firstSegmentName.indexOf("."));
- String afterDecimal = firstSegmentName.substring(firstSegmentName.indexOf(".") + 1);
+ if (segmentToBeMerged.contains(".")) {
+ String beforeDecimal = segmentToBeMerged.substring(0, segmentToBeMerged.indexOf("."));
+ String afterDecimal = segmentToBeMerged.substring(segmentToBeMerged.indexOf(".") + 1);
int fraction = Integer.parseInt(afterDecimal) + 1;
- String mergedSegmentName = beforeDecimal + "." + fraction;
- return mergedSegmentName;
+ return beforeDecimal + "." + fraction;
} else {
- String mergeName = firstSegmentName + "." + 1;
- return mergeName;
+ return segmentToBeMerged + "." + 1;
}
}
@@ -225,9 +222,9 @@ public final class CarbonDataMergerUtil {
for (String compactedBlocks : updatedDeltaFilesList) {
// Try to BlockName
- String fullBlock = compactedBlocks;
- int endIndex = fullBlock.lastIndexOf(File.separator);
- String blkNoExt = fullBlock.substring(endIndex + 1, fullBlock.lastIndexOf("-"));
+ int endIndex = compactedBlocks.lastIndexOf(File.separator);
+ String blkNoExt =
+ compactedBlocks.substring(endIndex + 1, compactedBlocks.lastIndexOf("-"));
blockNames.add(blkNoExt);
}
@@ -422,10 +419,7 @@ public final class CarbonDataMergerUtil {
// Check for segments which are qualified for IUD compaction.
if (compactionType.equals(CompactionType.IUD_UPDDEL_DELTA_COMPACTION)) {
- List<LoadMetadataDetails> listOfSegmentsToBeMerged =
- identifySegmentsToBeMergedBasedOnIUD(sortedSegments, carbonLoadModel);
-
- return listOfSegmentsToBeMerged;
+ return identifySegmentsToBeMergedBasedOnIUD(sortedSegments, carbonLoadModel);
}
// check preserve property and preserve the configured number of latest loads.
@@ -987,8 +981,7 @@ public final class CarbonDataMergerUtil {
List<String> list = segmentUpdateStatusManager.getUpdateDeltaFiles(seg);
- String fullBlock = blkName;
- String[] FileParts = fullBlock.split(CarbonCommonConstants.FILE_SEPARATOR);
+ String[] FileParts = blkName.split(CarbonCommonConstants.FILE_SEPARATOR);
String blockName = FileParts[FileParts.length - 1];
for (String str : list) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/81149f65/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
index a8f9520..81ee408 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
@@ -344,12 +344,11 @@ public class CompactionResultSortProcessor extends AbstractResultProcessor {
* @return
*/
private SortParameters createSortParameters() {
- SortParameters parameters = SortParameters
+ return SortParameters
.createSortParameters(carbonLoadModel.getDatabaseName(), tableName, dimensionColumnCount,
segmentProperties.getComplexDimensions().size(), measureCount, noDictionaryCount,
carbonLoadModel.getPartitionId(), segmentId, carbonLoadModel.getTaskNo(),
noDictionaryColMapping, true);
- return parameters;
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/81149f65/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
index de469f9..0978a02 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
@@ -83,9 +83,7 @@ public final class DataLoadProcessBuilder {
AbstractDataLoadProcessorStep sortProcessorStep =
new SortProcessorStepImpl(configuration, converterProcessorStep);
// 4. Writes the sorted data in carbondata format.
- AbstractDataLoadProcessorStep writerProcessorStep =
- new DataWriterProcessorStepImpl(configuration, sortProcessorStep);
- return writerProcessorStep;
+ return new DataWriterProcessorStepImpl(configuration, sortProcessorStep);
}
private AbstractDataLoadProcessorStep buildInternalForNoSort(CarbonIterator[] inputIterators,
@@ -116,9 +114,7 @@ public final class DataLoadProcessBuilder {
AbstractDataLoadProcessorStep sortProcessorStep =
new SortProcessorStepImpl(configuration, converterProcessorStep);
// 4. Writes the sorted data in carbondata format.
- AbstractDataLoadProcessorStep writerProcessorStep =
- new DataWriterBatchProcessorStepImpl(configuration, sortProcessorStep);
- return writerProcessorStep;
+ return new DataWriterBatchProcessorStepImpl(configuration, sortProcessorStep);
}
private AbstractDataLoadProcessorStep buildInternalForBucketing(CarbonIterator[] inputIterators,
@@ -134,9 +130,7 @@ public final class DataLoadProcessBuilder {
AbstractDataLoadProcessorStep sortProcessorStep =
new SortProcessorStepImpl(configuration, converterProcessorStep);
// 4. Writes the sorted data in carbondata format.
- AbstractDataLoadProcessorStep writerProcessorStep =
- new DataWriterProcessorStepImpl(configuration, sortProcessorStep);
- return writerProcessorStep;
+ return new DataWriterProcessorStepImpl(configuration, sortProcessorStep);
}
private CarbonDataLoadConfiguration createConfiguration(CarbonLoadModel loadModel,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/81149f65/processing/src/main/java/org/apache/carbondata/processing/newflow/dictionary/DictionaryServerClientDictionary.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/dictionary/DictionaryServerClientDictionary.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/dictionary/DictionaryServerClientDictionary.java
index c7ff1c7..82605a5 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/dictionary/DictionaryServerClientDictionary.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/dictionary/DictionaryServerClientDictionary.java
@@ -84,7 +84,6 @@ public class DictionaryServerClientDictionary implements BiDictionary<Integer, O
@Override public int size() {
dictionaryMessage.setType(DictionaryMessageType.SIZE);
- int size = client.getDictionary(dictionaryMessage).getDictionaryValue() + base;
- return size;
+ return client.getDictionary(dictionaryMessage).getDictionaryValue() + base;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/81149f65/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterWithBucketingImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterWithBucketingImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterWithBucketingImpl.java
index 2db732c..4a70710 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterWithBucketingImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterWithBucketingImpl.java
@@ -143,13 +143,11 @@ public class ParallelReadMergeSorterWithBucketingImpl extends AbstractMergeSorte
// Set the data file location
String dataFolderLocation =
storeLocation + File.separator + CarbonCommonConstants.SORT_TEMP_FILE_LOCATION;
- SingleThreadFinalSortFilesMerger finalMerger =
- new SingleThreadFinalSortFilesMerger(dataFolderLocation, sortParameters.getTableName(),
+ return new SingleThreadFinalSortFilesMerger(dataFolderLocation, sortParameters.getTableName(),
sortParameters.getDimColCount(), sortParameters.getComplexDimColCount(),
sortParameters.getMeasureColCount(), sortParameters.getNoDictionaryCount(),
sortParameters.getAggType(), sortParameters.getNoDictionaryDimnesionColumn(),
this.sortParameters.getNoDictionarySortColumn());
- return finalMerger;
}
@Override public void close() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/81149f65/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
index 84aa029..cfdb69a 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
@@ -255,8 +255,7 @@ public class UnsafeSortTempFileChunkHolder implements SortTempChunkHolder {
prefetchRecordsProceesed++;
returnRow = currentBuffer[bufferRowCounter++];
} else {
- Object[] outRow = getRowFromStream();
- this.returnRow = outRow;
+ this.returnRow = getRowFromStream();
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/81149f65/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java
index 5d065b1..cfd0dc2 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java
@@ -151,13 +151,12 @@ public class DataConverterProcessorStepImpl extends AbstractDataLoadProcessorSte
}
CarbonTableIdentifier identifier =
configuration.getTableIdentifier().getCarbonTableIdentifier();
- BadRecordsLogger badRecordsLogger = new BadRecordsLogger(identifier.getBadRecordLoggerKey(),
+ return new BadRecordsLogger(identifier.getBadRecordLoggerKey(),
identifier.getTableName() + '_' + System.currentTimeMillis(), getBadLogStoreLocation(
identifier.getDatabaseName() + CarbonCommonConstants.FILE_SEPARATOR + identifier
.getTableName() + CarbonCommonConstants.FILE_SEPARATOR + configuration.getSegmentId()
+ CarbonCommonConstants.FILE_SEPARATOR + configuration.getTaskNo()),
badRecordsLogRedirect, badRecordsLoggerEnable, badRecordConvertNullDisable, isDataLoadFail);
- return badRecordsLogger;
}
private String getBadLogStoreLocation(String storeLocation) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/81149f65/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java
index 78df028..baf9bcf 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java
@@ -178,13 +178,12 @@ public class DataConverterProcessorWithBucketingStepImpl extends AbstractDataLoa
}
CarbonTableIdentifier identifier =
configuration.getTableIdentifier().getCarbonTableIdentifier();
- BadRecordsLogger badRecordsLogger = new BadRecordsLogger(identifier.getBadRecordLoggerKey(),
+ return new BadRecordsLogger(identifier.getBadRecordLoggerKey(),
identifier.getTableName() + '_' + System.currentTimeMillis(), getBadLogStoreLocation(
identifier.getDatabaseName() + CarbonCommonConstants.FILE_SEPARATOR + identifier
.getTableName() + CarbonCommonConstants.FILE_SEPARATOR + configuration.getSegmentId()
+ CarbonCommonConstants.FILE_SEPARATOR + configuration.getTaskNo()),
badRecordsLogRedirect, badRecordsLoggerEnable, badRecordConvertNullDisable, isDataLoadFail);
- return badRecordsLogger;
}
private String getBadLogStoreLocation(String storeLocation) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/81149f65/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/SortProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/SortProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/SortProcessorStepImpl.java
index 698459c..b9beec0 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/SortProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/SortProcessorStepImpl.java
@@ -58,8 +58,7 @@ public class SortProcessorStepImpl extends AbstractDataLoadProcessorStep {
@Override
public Iterator<CarbonRowBatch>[] execute() throws CarbonDataLoadingException {
final Iterator<CarbonRowBatch>[] iterators = child.execute();
- Iterator<CarbonRowBatch>[] sortedIterators = sorter.sort(iterators);
- return sortedIterators;
+ return sorter.sort(iterators);
}
@Override
http://git-wip-us.apache.org/repos/asf/carbondata/blob/81149f65/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortTempFileChunkHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortTempFileChunkHolder.java b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortTempFileChunkHolder.java
index 2b0b8ae..6695a5b 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortTempFileChunkHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortTempFileChunkHolder.java
@@ -267,8 +267,7 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
prefetchRecordsProceesed++;
returnRow = currentBuffer[bufferRowCounter++];
} else {
- Object[] outRow = getRowFromStream();
- this.returnRow = outRow;
+ this.returnRow = getRowFromStream();
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/81149f65/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
index 61322bc..6fd29d7 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
@@ -198,8 +198,6 @@ public class CarbonFactDataHandlerModel {
CarbonTableIdentifier identifier =
configuration.getTableIdentifier().getCarbonTableIdentifier();
- CarbonTableIdentifier tableIdentifier =
- identifier;
boolean[] isUseInvertedIndex =
CarbonDataProcessorUtil.getIsUseInvertedIndex(configuration.getDataFields());
@@ -232,11 +230,11 @@ public class CarbonFactDataHandlerModel {
}
CarbonTable carbonTable = CarbonMetadata.getInstance().getCarbonTable(
- tableIdentifier.getDatabaseName() + CarbonCommonConstants.UNDERSCORE + tableIdentifier
+ identifier.getDatabaseName() + CarbonCommonConstants.UNDERSCORE + identifier
.getTableName());
List<ColumnSchema> wrapperColumnSchema = CarbonUtil
- .getColumnSchemaList(carbonTable.getDimensionByTableName(tableIdentifier.getTableName()),
- carbonTable.getMeasureByTableName(tableIdentifier.getTableName()));
+ .getColumnSchemaList(carbonTable.getDimensionByTableName(identifier.getTableName()),
+ carbonTable.getMeasureByTableName(identifier.getTableName()));
int[] colCardinality =
CarbonUtil.getFormattedCardinality(dimLensWithComplex, wrapperColumnSchema);
SegmentProperties segmentProperties =
@@ -388,10 +386,8 @@ public class CarbonFactDataHandlerModel {
.getTableName());
CarbonTablePath carbonTablePath =
CarbonStorePath.getCarbonTablePath(carbonStorePath, carbonTable.getCarbonTableIdentifier());
- String carbonDataDirectoryPath = carbonTablePath
- .getCarbonDataDirectoryPath(configuration.getPartitionId(),
- configuration.getSegmentId() + "");
- return carbonDataDirectoryPath;
+ return carbonTablePath.getCarbonDataDirectoryPath(configuration.getPartitionId(),
+ configuration.getSegmentId() + "");
}
public int[] getColCardinality() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/81149f65/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
index 41bfbed..c6eeb4c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
@@ -203,8 +203,7 @@ public final class CarbonDataProcessorUtil {
CarbonStorePath.getCarbonTablePath(baseStorePath, carbonTable.getCarbonTableIdentifier());
String carbonDataDirectoryPath =
carbonTablePath.getCarbonDataDirectoryPath(partitionId, segmentId + "");
- String localDataLoadFolderLocation = carbonDataDirectoryPath + File.separator + taskId;
- return localDataLoadFolderLocation;
+ return carbonDataDirectoryPath + File.separator + taskId;
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/81149f65/processing/src/main/java/org/apache/carbondata/processing/util/NonDictionaryUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/NonDictionaryUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/NonDictionaryUtil.java
index 54b4951..d6f992c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/NonDictionaryUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/NonDictionaryUtil.java
@@ -404,8 +404,7 @@ public class NonDictionaryUtil {
builder.append(CarbonCommonConstants.COMA_SPC_CHARACTER);
}
int lastIndex = builder.lastIndexOf(CarbonCommonConstants.COMA_SPC_CHARACTER);
- String str = -1 != lastIndex ? builder.substring(0, lastIndex) : builder.toString();
- return str;
+ return -1 != lastIndex ? builder.substring(0, lastIndex) : builder.toString();
}
/**
[18/19] carbondata git commit: fix rebase issue
Posted by ra...@apache.org.
fix rebase issue
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/fa08ab43
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/fa08ab43
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/fa08ab43
Branch: refs/heads/master
Commit: fa08ab43af1fb11712962e46817bbfec834a2b52
Parents: b6b93f0
Author: QiangCai <qi...@qq.com>
Authored: Sun May 14 19:13:53 2017 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Sun May 14 20:40:14 2017 +0800
----------------------------------------------------------------------
.../spark/testsuite/dataload/TestLoadDataFrame.scala | 4 ++--
.../processing/newflow/sort/unsafe/UnsafeSortDataRows.java | 6 ++++--
2 files changed, 6 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fa08ab43/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala
index 9179c08..994acf6 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala
@@ -171,7 +171,7 @@ class TestLoadDataFrame extends QueryTest with BeforeAndAfterAll {
test("test load dataframe with single pass enabled") {
// save dataframe to carbon file
- df.write
+ df2.write
.format("carbondata")
.option("tableName", "carbon8")
.option("tempCSV", "false")
@@ -186,7 +186,7 @@ class TestLoadDataFrame extends QueryTest with BeforeAndAfterAll {
test("test load dataframe with single pass disabled") {
// save dataframe to carbon file
- df.write
+ df2.write
.format("carbondata")
.option("tableName", "carbon9")
.option("tempCSV", "true")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fa08ab43/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeSortDataRows.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeSortDataRows.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeSortDataRows.java
index 0acc6a1..94a2d07 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeSortDataRows.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeSortDataRows.java
@@ -216,9 +216,11 @@ public class UnsafeSortDataRows {
dataSorterAndWriterExecutorService.submit(new DataSorterAndWriter(rowPage));
MemoryBlock memoryBlock = getMemoryBlock(inMemoryChunkSize);
boolean saveToDisk = !UnsafeMemoryManager.INSTANCE.isMemoryAvailable();
- rowPage = new UnsafeCarbonRowPage(parameters.getNoDictionaryDimnesionColumn(),
+ rowPage = new UnsafeCarbonRowPage(
+ parameters.getNoDictionaryDimnesionColumn(),
+ parameters.getNoDictionarySortColumn(),
parameters.getDimColCount(), parameters.getMeasureColCount(),
- parameters.getAggType(), memoryBlock,
+ parameters.getMeasureDataType(), memoryBlock,
saveToDisk);
rowPage.addRow(row);
} catch (Exception e) {