You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2017/09/01 07:16:30 UTC
[1/2] carbondata git commit: Fix Findbugs [Forced Update!]
Repository: carbondata
Updated Branches:
refs/heads/branch-1.2 770e5f46b -> baca6f925 (forced update)
Fix Findbugs
This closes #1287
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/bb0b3471
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/bb0b3471
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/bb0b3471
Branch: refs/heads/branch-1.2
Commit: bb0b34717db38a4bba15b39351895a2492d9e48c
Parents: 2be683e
Author: Raghunandan S <ca...@gmail.com>
Authored: Thu Aug 24 01:22:57 2017 +0530
Committer: Ravindra Pesala <ra...@gmail.com>
Committed: Fri Sep 1 11:01:39 2017 +0530
----------------------------------------------------------------------
.../DictionaryColumnUniqueIdentifier.java | 2 +
.../page/encoding/DefaultEncodingStrategy.java | 1 +
.../adaptive/AdaptiveDeltaIntegralCodec.java | 1 +
.../adaptive/AdaptiveFloatingCodec.java | 1 +
.../adaptive/AdaptiveIntegralCodec.java | 1 +
.../encoding/compress/DirectCompressCodec.java | 3 +-
.../dimension/legacy/IndexStorageCodec.java | 3 --
.../datastore/page/encoding/rle/RLECodec.java | 1 +
.../page/statistics/LVStringStatsCollector.java | 2 +-
.../carbondata/core/scan/filter/FilterUtil.java | 11 +++--
.../visitor/DictionaryColumnVisitor.java | 49 ++++++++++----------
.../visitor/RangeDictionaryColumnVisitor.java | 30 ++++++------
.../carbondata/core/util/TaskMetricsMap.java | 4 +-
.../carbondata/presto/PrestoFilterUtil.java | 3 +-
.../presto/impl/CarbonTableCacheModel.java | 1 -
.../VectorizedCarbonRecordReader.java | 2 +-
.../carbondata/processing/store/TablePage.java | 3 ++
17 files changed, 66 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bb0b3471/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifier.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifier.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifier.java
index f0182e4..3effe8a 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifier.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifier.java
@@ -29,6 +29,8 @@ import org.apache.carbondata.core.util.path.CarbonTablePath;
*/
public class DictionaryColumnUniqueIdentifier implements Serializable {
+ private static final long serialVersionUID = -1231234567L;
+
/**
* table fully qualified name
*/
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bb0b3471/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DefaultEncodingStrategy.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DefaultEncodingStrategy.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DefaultEncodingStrategy.java
index b024888..04ca8a3 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DefaultEncodingStrategy.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DefaultEncodingStrategy.java
@@ -51,6 +51,7 @@ public class DefaultEncodingStrategy extends EncodingStrategy {
if (newWay) {
return createEncoderForDimension((TableSpec.DimensionSpec) columnSpec, inputPage);
} else {
+ assert columnSpec instanceof TableSpec.DimensionSpec;
return createEncoderForDimensionLegacy((TableSpec.DimensionSpec) columnSpec);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bb0b3471/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaIntegralCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaIntegralCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaIntegralCodec.java
index 128c66b..60ff3ab 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaIntegralCodec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaIntegralCodec.java
@@ -112,6 +112,7 @@ public class AdaptiveDeltaIntegralCodec extends AdaptiveCodec {
@Override
public ColumnPageDecoder createDecoder(ColumnPageEncoderMeta meta) {
+ assert meta instanceof AdaptiveDeltaIntegralEncoderMeta;
AdaptiveDeltaIntegralEncoderMeta codecMeta = (AdaptiveDeltaIntegralEncoderMeta) meta;
final Compressor compressor = CompressorFactory.getInstance().getCompressor(
codecMeta.getCompressorName());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bb0b3471/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java
index 9b03049..b86f76f 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java
@@ -96,6 +96,7 @@ public class AdaptiveFloatingCodec extends AdaptiveCodec {
@Override
public ColumnPageDecoder createDecoder(ColumnPageEncoderMeta meta) {
+ assert meta instanceof AdaptiveFloatingEncoderMeta;
AdaptiveFloatingEncoderMeta codecMeta = (AdaptiveFloatingEncoderMeta) meta;
final Compressor compressor = CompressorFactory.getInstance().getCompressor(
codecMeta.getCompressorName());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bb0b3471/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveIntegralCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveIntegralCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveIntegralCodec.java
index ccd92d1..3688102 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveIntegralCodec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveIntegralCodec.java
@@ -86,6 +86,7 @@ public class AdaptiveIntegralCodec extends AdaptiveCodec {
@Override
public ColumnPageDecoder createDecoder(ColumnPageEncoderMeta meta) {
+ assert meta instanceof AdaptiveIntegralEncoderMeta;
AdaptiveIntegralEncoderMeta codecMeta = (AdaptiveIntegralEncoderMeta) meta;
final Compressor compressor = CompressorFactory.getInstance().getCompressor(
codecMeta.getCompressorName());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bb0b3471/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/compress/DirectCompressCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/compress/DirectCompressCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/compress/DirectCompressCodec.java
index 1fc61ac..cb1508f 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/compress/DirectCompressCodec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/compress/DirectCompressCodec.java
@@ -60,12 +60,13 @@ public class DirectCompressCodec implements ColumnPageCodec {
@Override
public ColumnPageDecoder createDecoder(ColumnPageEncoderMeta meta) {
+ assert meta instanceof DirectCompressorEncoderMeta;
DirectCompressorEncoderMeta codecMeta = (DirectCompressorEncoderMeta) meta;
return new DirectDecompressor(codecMeta.getCompressorName(),
codecMeta.getScale(), codecMeta.getPrecision());
}
- private class DirectCompressor extends ColumnPageEncoder {
+ private static class DirectCompressor extends ColumnPageEncoder {
private Compressor compressor;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bb0b3471/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/dimension/legacy/IndexStorageCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/dimension/legacy/IndexStorageCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/dimension/legacy/IndexStorageCodec.java
index d228f6e..cb6b387 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/dimension/legacy/IndexStorageCodec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/dimension/legacy/IndexStorageCodec.java
@@ -21,11 +21,8 @@ import org.apache.carbondata.core.datastore.compression.Compressor;
import org.apache.carbondata.core.datastore.page.encoding.ColumnPageCodec;
import org.apache.carbondata.core.datastore.page.encoding.ColumnPageDecoder;
import org.apache.carbondata.core.datastore.page.encoding.ColumnPageEncoderMeta;
-import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
-import org.apache.carbondata.core.util.CarbonProperties;
public abstract class IndexStorageCodec implements ColumnPageCodec {
- protected ColumnarFormatVersion version = CarbonProperties.getInstance().getFormatVersion();
protected Compressor compressor;
protected boolean isSort;
protected boolean isInvertedIndex;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bb0b3471/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/rle/RLECodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/rle/RLECodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/rle/RLECodec.java
index a2837b8..12690a5 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/rle/RLECodec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/rle/RLECodec.java
@@ -62,6 +62,7 @@ public class RLECodec implements ColumnPageCodec {
@Override
public ColumnPageDecoder createDecoder(ColumnPageEncoderMeta meta) {
+ assert meta instanceof RLEEncoderMeta;
RLEEncoderMeta codecMeta = (RLEEncoderMeta) meta;
return new RLEDecoder(codecMeta.getDataType(), codecMeta.getPageSize());
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bb0b3471/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/LVStringStatsCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/LVStringStatsCollector.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/LVStringStatsCollector.java
index 98a757f..1b26a60 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/LVStringStatsCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/LVStringStatsCollector.java
@@ -81,7 +81,7 @@ public class LVStringStatsCollector implements ColumnPageStatsCollector {
}
return;
}
- int length = (value[0] << 8) + value[1];
+ int length = (value[0] << 8) + (value[1] & 0xff);
assert (length > 0);
byte[] v = new byte[value.length - 2];
System.arraycopy(value, 2, v, 0, v.length);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bb0b3471/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
index 39dee5a..78c1afd 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
@@ -1107,10 +1107,13 @@ public final class FilterUtil {
*/
public static Dictionary getForwardDictionaryCache(AbsoluteTableIdentifier tableIdentifier,
CarbonDimension carbonDimension, TableProvider tableProvider) throws IOException {
- CarbonTable carbonTable =
- tableProvider.getCarbonTable(tableIdentifier.getCarbonTableIdentifier());
- CarbonTablePath carbonTablePath =
- CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier());
+ CarbonTablePath carbonTablePath = null;
+ if (null != tableProvider) {
+ CarbonTable carbonTable =
+ tableProvider.getCarbonTable(tableIdentifier.getCarbonTableIdentifier());
+ carbonTablePath =
+ CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier());
+ }
DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier =
new DictionaryColumnUniqueIdentifier(tableIdentifier.getCarbonTableIdentifier(),
carbonDimension.getColumnIdentifier(), carbonDimension.getDataType(), carbonTablePath);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bb0b3471/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java
index 451ce45..47f1c8f 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java
@@ -44,31 +44,32 @@ public class DictionaryColumnVisitor implements ResolvedFilterInfoVisitorIntf {
*/
public void populateFilterResolvedInfo(ColumnResolvedFilterInfo visitableObj,
FilterResolverMetadata metadata) throws FilterUnsupportedException, IOException {
-
- DimColumnResolvedFilterInfo resolveDimension = (DimColumnResolvedFilterInfo) visitableObj;
- ColumnFilterInfo resolvedFilterObject = null;
- List<String> evaluateResultListFinal;
- try {
- evaluateResultListFinal = metadata.getExpression().evaluate(null).getListAsString();
- } catch (FilterIllegalMemberException e) {
- throw new FilterUnsupportedException(e);
- }
- resolvedFilterObject = FilterUtil
- .getFilterValues(metadata.getTableIdentifier(), metadata.getColumnExpression(),
- evaluateResultListFinal, metadata.isIncludeFilter(), metadata.getTableProvider());
- if (!metadata.isIncludeFilter() && null != resolvedFilterObject) {
- // Adding default surrogate key of null member inorder to not display the same while
- // displaying the report as per hive compatibility.
- // first check of surrogate key for null value is already added then
- // no need to add again otherwise result will be wrong in case of exclude filter
- // this is because two times it will flip the same bit
- if (!resolvedFilterObject.getFilterList()
- .contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY)) {
- resolvedFilterObject.getFilterList()
- .add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY);
+ if (visitableObj instanceof DimColumnResolvedFilterInfo) {
+ DimColumnResolvedFilterInfo resolveDimension = (DimColumnResolvedFilterInfo) visitableObj;
+ ColumnFilterInfo resolvedFilterObject = null;
+ List<String> evaluateResultListFinal;
+ try {
+ evaluateResultListFinal = metadata.getExpression().evaluate(null).getListAsString();
+ } catch (FilterIllegalMemberException e) {
+ throw new FilterUnsupportedException(e);
+ }
+ resolvedFilterObject = FilterUtil
+ .getFilterValues(metadata.getTableIdentifier(), metadata.getColumnExpression(),
+ evaluateResultListFinal, metadata.isIncludeFilter(), metadata.getTableProvider());
+ if (!metadata.isIncludeFilter() && null != resolvedFilterObject) {
+ // Adding default surrogate key of null member inorder to not display the same while
+ // displaying the report as per hive compatibility.
+ // first check of surrogate key for null value is already added then
+ // no need to add again otherwise result will be wrong in case of exclude filter
+ // this is because two times it will flip the same bit
+ if (!resolvedFilterObject.getFilterList()
+ .contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY)) {
+ resolvedFilterObject.getFilterList()
+ .add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY);
+ }
+ Collections.sort(resolvedFilterObject.getFilterList());
}
- Collections.sort(resolvedFilterObject.getFilterList());
+ resolveDimension.setFilterValues(resolvedFilterObject);
}
- resolveDimension.setFilterValues(resolvedFilterObject);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bb0b3471/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDictionaryColumnVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDictionaryColumnVisitor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDictionaryColumnVisitor.java
index 5ce818d..40a5430 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDictionaryColumnVisitor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDictionaryColumnVisitor.java
@@ -44,21 +44,23 @@ public class RangeDictionaryColumnVisitor extends DictionaryColumnVisitor
*/
public void populateFilterResolvedInfo(ColumnResolvedFilterInfo visitableObj,
FilterResolverMetadata metadata) throws FilterUnsupportedException, IOException {
- DimColumnResolvedFilterInfo resolveDimension = (DimColumnResolvedFilterInfo) visitableObj;
- ColumnFilterInfo resolvedFilterObject = null;
- List<String> evaluateResultListFinal;
- resolvedFilterObject = FilterUtil
- .getFilterListForAllValues(metadata.getTableIdentifier(), metadata.getExpression(),
- metadata.getColumnExpression(), metadata.isIncludeFilter(),
- metadata.getTableProvider());
+ if (visitableObj instanceof DimColumnResolvedFilterInfo) {
+ DimColumnResolvedFilterInfo resolveDimension = (DimColumnResolvedFilterInfo) visitableObj;
+ ColumnFilterInfo resolvedFilterObject = null;
+ List<String> evaluateResultListFinal;
+ resolvedFilterObject = FilterUtil
+ .getFilterListForAllValues(metadata.getTableIdentifier(), metadata.getExpression(),
+ metadata.getColumnExpression(), metadata.isIncludeFilter(),
+ metadata.getTableProvider());
- if (!metadata.isIncludeFilter() && null != resolvedFilterObject) {
- // Adding default surrogate key of null member inorder to not display the same while
- // displaying the report as per hive compatibility.
- resolvedFilterObject.getFilterList()
- .add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY);
- Collections.sort(resolvedFilterObject.getFilterList());
+ if (!metadata.isIncludeFilter() && null != resolvedFilterObject) {
+ // Adding default surrogate key of null member inorder to not display the same while
+ // displaying the report as per hive compatibility.
+ resolvedFilterObject.getFilterList()
+ .add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY);
+ Collections.sort(resolvedFilterObject.getFilterList());
+ }
+ resolveDimension.setFilterValues(resolvedFilterObject);
}
- resolveDimension.setFilterValues(resolvedFilterObject);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bb0b3471/core/src/main/java/org/apache/carbondata/core/util/TaskMetricsMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/TaskMetricsMap.java b/core/src/main/java/org/apache/carbondata/core/util/TaskMetricsMap.java
index 0dc6c97..bd2d4c2 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/TaskMetricsMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/TaskMetricsMap.java
@@ -41,10 +41,10 @@ public class TaskMetricsMap {
* In this map we are maintaining all spawned child threads callback info for each parent thread
* here key = parent thread id & values = list of spawned child threads callbacks
*/
- public static Map<Long, List<CarbonFSBytesReadOnThreadCallback>> metricMap =
+ public static final Map<Long, List<CarbonFSBytesReadOnThreadCallback>> metricMap =
new ConcurrentHashMap<>();
- public static TaskMetricsMap taskMetricsMap = new TaskMetricsMap();
+ public static final TaskMetricsMap taskMetricsMap = new TaskMetricsMap();
public static TaskMetricsMap getInstance() {
return taskMetricsMap;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bb0b3471/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoFilterUtil.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoFilterUtil.java b/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoFilterUtil.java
index f665d9d..9a5a5cb 100644
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoFilterUtil.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoFilterUtil.java
@@ -205,7 +205,8 @@ public class PrestoFilterUtil {
}
private static Object ConvertDataByType(Object rawdata, Type type) {
- if (type.equals(IntegerType.INTEGER)) return new Integer((rawdata.toString()));
+ if (type.equals(IntegerType.INTEGER)) return Integer.valueOf(rawdata.toString());
+ // new Integer((rawdata.toString()));
else if (type.equals(BigintType.BIGINT)) return rawdata;
else if (type.equals(VarcharType.VARCHAR)) {
if (rawdata instanceof Slice) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bb0b3471/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableCacheModel.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableCacheModel.java b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableCacheModel.java
index 2a4db14..8422c3e 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableCacheModel.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableCacheModel.java
@@ -33,7 +33,6 @@ public class CarbonTableCacheModel {
public TableInfo tableInfo;
public CarbonTable carbonTable;
- public String[] segments;
public boolean isValid() {
if (carbonTable != null && carbonTablePath != null && carbonTableIdentifier != null)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bb0b3471/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java b/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java
index 4914d8d..5356ad5 100644
--- a/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java
+++ b/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java
@@ -155,7 +155,7 @@ class VectorizedCarbonRecordReader extends AbstractRecordReader<Object> {
if (returnColumnarBatch) {
int value = columnarBatch.numValidRows();
rowCount += value;
- inputMetricsStats.incrementRecordRead(new Long(value));
+ inputMetricsStats.incrementRecordRead((long)value);
return columnarBatch;
}
rowCount += 1;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bb0b3471/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java b/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java
index d403a93..ab0a122 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java
@@ -307,6 +307,9 @@ public class TablePage {
complexDimensionPages[complexDimIndex++]);
encodedComplexDimenions.addAll(Arrays.asList(encodedPages));
break;
+ default:
+ throw new IllegalArgumentException("unsupported dimension type:" + spec
+ .getDimensionType());
}
}
[2/2] carbondata git commit: [CARBONDATA-1443] Schema storage has
issue while converting table object to json string
Posted by ra...@apache.org.
[CARBONDATA-1443] Schema storage has issue while converting table object to json string
Every table object converts to json and splits to small splits as there is limit in hive metastore of 4000. While splitting the json string code has problem so it could not get table and throws NPE
This closes #1314
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/baca6f92
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/baca6f92
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/baca6f92
Branch: refs/heads/branch-1.2
Commit: baca6f925ad973864ff432dbaccd6a99df5ea363
Parents: bb0b347
Author: Ravindra Pesala <ra...@gmail.com>
Authored: Fri Sep 1 10:51:04 2017 +0530
Committer: chenliang613 <ch...@apache.org>
Committed: Fri Sep 1 14:25:00 2017 +0800
----------------------------------------------------------------------
.../apache/carbondata/core/util/CarbonUtil.java | 51 +++++++++++++++++---
.../carbondata/core/util/CarbonUtilTest.java | 47 +++++++++++++++++-
2 files changed, 91 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/baca6f92/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 67c1d81..8b6e44a 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -1803,10 +1803,31 @@ public final class CarbonUtil {
return !(null == badRecordsLocation || badRecordsLocation.length() == 0);
}
+ /**
+ * Converts Tableinfo object to json multi string objects of size 4000
+ * @param tableInfo
+ * @param seperator separator between each string
+ * @param quote Quote to be used for string
+ * @param prefix Prefix to be added before generated string
+ * @return
+ */
public static String convertToMultiGsonStrings(TableInfo tableInfo, String seperator,
String quote, String prefix) {
Gson gson = new Gson();
String schemaString = gson.toJson(tableInfo);
+ return splitSchemaStringToMultiString(seperator, quote, prefix, schemaString);
+ }
+
+ /**
+ * Converts Json String to multi string objects of size 4000
+ * @param schemaString Json string
+ * @param seperator separator between each string
+ * @param quote Quote to be used for string
+ * @param prefix Prefix to be added before generated string
+ * @return
+ */
+ public static String splitSchemaStringToMultiString(String seperator, String quote,
+ String prefix, String schemaString) {
int schemaLen = schemaString.length();
int splitLen = 4000;
int parts = schemaLen / splitLen;
@@ -1817,10 +1838,12 @@ public final class CarbonUtil {
new StringBuilder(prefix).append(quote).append("carbonSchemaPartsNo").append(quote)
.append(seperator).append("'").append(parts).append("',");
int runningLen = 0;
- int endLen = splitLen;
+ int endLen = schemaLen > splitLen ? splitLen : schemaLen;
for (int i = 0; i < parts; i++) {
if (i == parts - 1) {
- endLen = schemaLen % splitLen;
+ if (schemaLen % splitLen > 0) {
+ endLen = schemaLen % splitLen;
+ }
}
builder.append(quote).append("carbonSchema").append(i).append(quote).append(seperator);
builder.append("'").append(schemaString.substring(runningLen, runningLen + endLen))
@@ -1833,10 +1856,25 @@ public final class CarbonUtil {
return builder.toString();
}
+ /**
+ * Converts Tableinfo object to json multi string objects of size 4000 and stored in map
+ * @param tableInfo
+ * @return
+ */
public static Map<String, String> convertToMultiStringMap(TableInfo tableInfo) {
- Map<String, String> map = new HashMap<>();
Gson gson = new Gson();
String schemaString = gson.toJson(tableInfo);
+ return splitSchemaStringToMap(schemaString);
+ }
+
+ /**
+ * Converts Json string to multi string objects of size 4000 and stored in map
+ *
+ * @param schemaString
+ * @return
+ */
+ public static Map<String, String> splitSchemaStringToMap(String schemaString) {
+ Map<String, String> map = new HashMap<>();
int schemaLen = schemaString.length();
int splitLen = 4000;
int parts = schemaLen / splitLen;
@@ -1845,10 +1883,12 @@ public final class CarbonUtil {
}
map.put("carbonSchemaPartsNo", parts + "");
int runningLen = 0;
- int endLen = splitLen;
+ int endLen = schemaLen > splitLen ? splitLen : schemaLen;
for (int i = 0; i < parts; i++) {
if (i == parts - 1) {
- endLen = schemaLen % splitLen;
+ if (schemaLen % splitLen > 0) {
+ endLen = schemaLen % splitLen;
+ }
}
map.put("carbonSchema" + i, schemaString.substring(runningLen, runningLen + endLen));
runningLen += splitLen;
@@ -1856,7 +1896,6 @@ public final class CarbonUtil {
return map;
}
-
public static TableInfo convertGsonToTableInfo(Map<String, String> properties) {
Gson gson = new Gson();
String partsNo = properties.get("carbonSchemaPartsNo");
http://git-wip-us.apache.org/repos/asf/carbondata/blob/baca6f92/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
index 776059f..4a8483f 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
@@ -27,6 +27,7 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.Map;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
@@ -1002,6 +1003,50 @@ public class CarbonUtilTest {
}
+ @Test
+ public void testSplitSchemaStringToMapWithLessThanSplitLen() {
+ String schema = generateString(399);
+ Map<String, String> map = CarbonUtil.splitSchemaStringToMap(schema);
+ assert (map.size() == 2);
+ String schemaString = CarbonUtil.splitSchemaStringToMultiString(" ", "'", ",", schema);
+ assert (schemaString.length() > schema.length());
+ }
+
+ @Test
+ public void testSplitSchemaStringToMapWithEqualThanSplitLen() {
+ String schema = generateString(4000);
+ Map<String, String> map = CarbonUtil.splitSchemaStringToMap(schema);
+ assert (map.size() == 2);
+ String schemaString = CarbonUtil.splitSchemaStringToMultiString(" ", "'", ",", schema);
+ assert (schemaString.length() > schema.length());
+ }
+
+ @Test
+ public void testSplitSchemaStringToMapWithMoreThanSplitLen() {
+ String schema = generateString(7999);
+ Map<String, String> map = CarbonUtil.splitSchemaStringToMap(schema);
+ assert (map.size() == 3);
+ String schemaString = CarbonUtil.splitSchemaStringToMultiString(" ", "'", ",", schema);
+ assert (schemaString.length() > schema.length());
+ }
+
+ @Test
+ public void testSplitSchemaStringToMapWithMultiplesOfSplitLen() {
+ String schema = generateString(12000);
+ Map<String, String> map = CarbonUtil.splitSchemaStringToMap(schema);
+ assert (map.size() == 4);
+ String schemaString = CarbonUtil.splitSchemaStringToMultiString(" ", "'", ",", schema);
+ assert (schemaString.length() > schema.length());
+ }
+
+ private String generateString(int length) {
+ StringBuilder builder = new StringBuilder();
+ for (int i = 0; i < length; i++) {
+ builder.append("a");
+ }
+ return builder.toString();
+ }
+
private void assertRangeIndex(byte[][] dataArr, byte[] dataChunk,
FixedLengthDimensionDataChunk fixedLengthDimensionDataChunk, byte[] keyWord, int[] expectRangeIndex) {
int[] range;
@@ -1013,7 +1058,7 @@ public class CarbonUtilTest {
int index = CarbonUtil.binarySearch(dataArr, 0, dataChunk.length / keyWord.length - 1, keyWord);
assertTrue(expectRangeIndex[0] <= index && index <= range[1]);
}
-
+
@AfterClass public static void testcleanUp() {
new File("../core/src/test/resources/testFile.txt").deleteOnExit();