You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/08/19 01:53:09 UTC
[1/3] carbondata git commit: [Review][CARBONDATA-1386] fixed findbugs
errors in carbondata-core
Repository: carbondata
Updated Branches:
refs/heads/master 2ee777551 -> 500654e60
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/dev/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/dev/findbugs-exclude.xml b/dev/findbugs-exclude.xml
index ccb8a1a..1520cd4 100644
--- a/dev/findbugs-exclude.xml
+++ b/dev/findbugs-exclude.xml
@@ -27,50 +27,33 @@
<!-- This method creates stream but the caller methods are responsible for closing the stream -->
<Match>
<Class name="org.apache.carbondata.core.datastore.impl.FileFactory"/>
- <Or>
- <Method name="getDataInputStream"/>
- </Or>
+ <Method name="getDataInputStream"/>
<Bug pattern="OBL_UNSATISFIED_OBLIGATION_EXCEPTION_EDGE"/>
</Match>
-
<Match>
- <Class name="org.apache.carbondata.core.scan.aggregator.impl.BitSet"/>
- <Bug pattern="SE_TRANSIENT_FIELD_NOT_RESTORED"/>
+ <Class name="org.apache.carbondata.core.datastore.impl.FileHolderImpl"/>
+ <Method name="getDataInputStream"/>
+ <Bug pattern="OBL_UNSATISFIED_OBLIGATION_EXCEPTION_EDGE"/>
</Match>
-
<Match>
- <Class name="org.apache.carbondata.core.scan.expression.ExpressionResult"/>
- <Or>
- <Method name="getBoolean"/>
- </Or>
- <Bug pattern="NP_BOOLEAN_RETURN_NULL"/>
+ <Class name="org.apache.carbondata.core.constants.CarbonCommonConstants"/>
+ <Bug pattern="MS_MUTABLE_ARRAY"/>
</Match>
-
<Match>
- <Class name="org.apache.carbondata.processing.csvreaderstep.BlockDataHandler"/>
- <Or>
- <Method name="readOneRow"/>
- </Or>
- <Bug pattern="NP_GUARANTEED_DEREF"/>
+ <Class name="org.apache.carbondata.core.scan.expression.ExpressionResult"/>
+ <Method name="getBoolean"/>
+ <Bug pattern="NP_BOOLEAN_RETURN_NULL"/>
</Match>
-
<Match>
- <!--
- A mutable static field could be changed by malicious code or by accident. The field could be
- made package protected to avoid this vulnerability.
-
- We have a set of stuff like:
- public static final byte [] SKIP_ARRAY = new byte [ ] {'S', 'K', 'I', 'P'};
-
- Warning is not wrong, but difficult to avoid...
- !-->
- <Bug pattern="MS_PKGPROTECT"/>
+ <Class name="org.apache.carbondata.core.metadata.schema.table.column.CarbonImplicitDimension"/>
+ <Method name="equals"/>
+ <Bug pattern="EQ_OVERRIDING_EQUALS_NOT_SYMMETRIC"/>
</Match>
<Match>
- <Bug pattern="MS_OOI_PKGPROTECT"/>
+ <Class name="org.apache.carbondata.core.statusmanager.LoadMetadataDetails"/>
+ <Method name="getTimeStamp"/>
+ <Bug pattern="STCAL_INVOKE_ON_STATIC_DATE_FORMAT_INSTANCE"/>
</Match>
-
-
<Match>
<!--
Returning a reference to a mutable object value stored in one of the object's fields exposes
@@ -87,8 +70,6 @@
<Match>
<Bug pattern="EI_EXPOSE_REP2"/>
</Match>
-
-
<Match>
<!--
This class implements the Comparator interface. You should consider whether or not it should
@@ -99,81 +80,22 @@
!-->
<Bug pattern="SE_COMPARATOR_SHOULD_BE_SERIALIZABLE"/>
</Match>
-
- <Match>
-
- <!--
- This method performs synchronization an object that is an instance of a class from
- the java.util.concurrent package (or its subclasses). Instances of these classes have their own
- concurrency control mechanisms that are orthogonal to the synchronization provided by the Java
- keyword synchronized. For example, synchronizing on an AtomicBoolean will not prevent other
- threads from modifying the AtomicBoolean.
-
- Such code may be correct, but should be carefully reviewed and documented, and may confuse people
- who have to maintain the code at a later date.
-
- We do that all the time to save lock objects.
- !-->
- <Bug pattern="JLM_JSR166_UTILCONCURRENT_MONITORENTER"/>
- </Match>
-
- <Match>
- <!--
- Found a call to a method which will perform a byte to String (or String to byte) conversion,
- and will assume that the default platform encoding is suitable. This will cause the
- application behaviour to vary between platforms. Use an alternative API and specify a
- charset name or Charset object explicitly.
- !-->
- <Bug pattern="DM_DEFAULT_ENCODING"/>
- </Match>
-
<Match>
- <!--
- Invoking System.exit shuts down the entire Java virtual machine. This should only been
- done when it is appropriate. Such calls make it hard or impossible for your code to be
- invoked by other code. Consider throwing a RuntimeException instead.
-
- It's so bad that the reviews will catch all the wrong cases.
- !-->
- <Bug pattern="DM_EXIT"/>
+ <Class name="org.apache.carbondata.core.scan.aggregator.impl.BitSet"/>
+ <Or>
+ <Bug pattern="SE_TRANSIENT_FIELD_NOT_RESTORED"/>
+ </Or>
</Match>
-
<Match>
- <!--
- This method returns a value that is not checked. The return value should be checked since
- it can indicate an unusual or unexpected function execution. For example, the
- File.delete() method returns false if the file could not be successfully deleted
- (rather than throwing an Exception). If you don't check the result, you won't notice
- if the method invocation signals unexpected behavior by returning an atypical return
- value.
-
- It's so bad that the reviews will catch all the wrong cases.
- !-->
- <Bug pattern="RV_RETURN_VALUE_IGNORED_BAD_PRACTICE"/>
+ <Bug pattern="SIC_INNER_SHOULD_BE_STATIC_ANON"/>
</Match>
<Match>
- <Bug pattern="RV_RETURN_VALUE_IGNORED_INFERRED"/>
+ <Bug pattern="DP_DO_INSIDE_DO_PRIVILEGED"/>
</Match>
-
-
<Match>
- <!--
- This method contains a redundant check of a known non-null value against the constant null.
-
- Most of the time we're securing ourselves, does no much harm.
- !-->
- <Bug pattern="RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE"/>
+ <Bug pattern="UWF_FIELD_NOT_INITIALIZED_IN_CONSTRUCTOR"/>
</Match>
-
<Match>
- <!--
- A final static field references an array and can be accessed by malicious code or by
- accident from another package. This code can freely modify the contents of the array.
-
- We've got this all over the place... Cloning the array by security is not a general
- solution from a performance point of view
- !-->
- <Bug pattern="MS_MUTABLE_ARRAY"/>
+ <Bug pattern="JLM_JSR166_UTILCONCURRENT_MONITORENTER"/>
</Match>
-
</FindBugsFilter>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index c8eec62..f722c66 100644
--- a/pom.xml
+++ b/pom.xml
@@ -421,7 +421,7 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>findbugs-maven-plugin</artifactId>
- <version>3.0.0</version>
+ <version>3.0.4</version>
<configuration>
<excludeFilterFile>${dev.path}/findbugs-exclude.xml</excludeFilterFile>
<failOnError>true</failOnError>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeCarbonRowPage.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeCarbonRowPage.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeCarbonRowPage.java
index 367d8a0..20b60c9 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeCarbonRowPage.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeCarbonRowPage.java
@@ -96,15 +96,15 @@ public class UnsafeCarbonRowPage {
for (; dimCount < noDictionaryDimensionMapping.length; dimCount++) {
if (noDictionaryDimensionMapping[dimCount]) {
byte[] col = (byte[]) row[dimCount];
- CarbonUnsafe.unsafe
+ CarbonUnsafe.getUnsafe()
.putShort(baseObject, address + size, (short) col.length);
size += 2;
- CarbonUnsafe.unsafe.copyMemory(col, CarbonUnsafe.BYTE_ARRAY_OFFSET, baseObject,
+ CarbonUnsafe.getUnsafe().copyMemory(col, CarbonUnsafe.BYTE_ARRAY_OFFSET, baseObject,
address + size, col.length);
size += col.length;
} else {
int value = (int) row[dimCount];
- CarbonUnsafe.unsafe.putInt(baseObject, address + size, value);
+ CarbonUnsafe.getUnsafe().putInt(baseObject, address + size, value);
size += 4;
}
}
@@ -112,9 +112,9 @@ public class UnsafeCarbonRowPage {
// write complex dimensions here.
for (; dimCount < dimensionSize; dimCount++) {
byte[] col = (byte[]) row[dimCount];
- CarbonUnsafe.unsafe.putShort(baseObject, address + size, (short) col.length);
+ CarbonUnsafe.getUnsafe().putShort(baseObject, address + size, (short) col.length);
size += 2;
- CarbonUnsafe.unsafe.copyMemory(col, CarbonUnsafe.BYTE_ARRAY_OFFSET, baseObject,
+ CarbonUnsafe.getUnsafe().copyMemory(col, CarbonUnsafe.BYTE_ARRAY_OFFSET, baseObject,
address + size, col.length);
size += col.length;
}
@@ -128,31 +128,31 @@ public class UnsafeCarbonRowPage {
switch (measureDataType[mesCount]) {
case SHORT:
Short sval = (Short) value;
- CarbonUnsafe.unsafe.putShort(baseObject, address + size, sval);
+ CarbonUnsafe.getUnsafe().putShort(baseObject, address + size, sval);
size += 2;
break;
case INT:
Integer ival = (Integer) value;
- CarbonUnsafe.unsafe.putInt(baseObject, address + size, ival);
+ CarbonUnsafe.getUnsafe().putInt(baseObject, address + size, ival);
size += 4;
break;
case LONG:
Long val = (Long) value;
- CarbonUnsafe.unsafe.putLong(baseObject, address + size, val);
+ CarbonUnsafe.getUnsafe().putLong(baseObject, address + size, val);
size += 8;
break;
case DOUBLE:
Double doubleVal = (Double) value;
- CarbonUnsafe.unsafe.putDouble(baseObject, address + size, doubleVal);
+ CarbonUnsafe.getUnsafe().putDouble(baseObject, address + size, doubleVal);
size += 8;
break;
case DECIMAL:
BigDecimal decimalVal = (BigDecimal) value;
byte[] bigDecimalInBytes = DataTypeUtil.bigDecimalToByte(decimalVal);
- CarbonUnsafe.unsafe.putShort(baseObject, address + size,
+ CarbonUnsafe.getUnsafe().putShort(baseObject, address + size,
(short) bigDecimalInBytes.length);
size += 2;
- CarbonUnsafe.unsafe
+ CarbonUnsafe.getUnsafe()
.copyMemory(bigDecimalInBytes, CarbonUnsafe.BYTE_ARRAY_OFFSET, baseObject,
address + size, bigDecimalInBytes.length);
size += bigDecimalInBytes.length;
@@ -163,7 +163,7 @@ public class UnsafeCarbonRowPage {
unset(nullSetWords, mesCount);
}
}
- CarbonUnsafe.unsafe.copyMemory(nullSetWords, CarbonUnsafe.LONG_ARRAY_OFFSET, baseObject,
+ CarbonUnsafe.getUnsafe().copyMemory(nullSetWords, CarbonUnsafe.LONG_ARRAY_OFFSET, baseObject,
address + nullWordLoc, nullSetSize);
return size;
}
@@ -175,16 +175,16 @@ public class UnsafeCarbonRowPage {
Object baseObject = dataBlock.getBaseObject();
for (; dimCount < noDictionaryDimensionMapping.length; dimCount++) {
if (noDictionaryDimensionMapping[dimCount]) {
- short aShort = CarbonUnsafe.unsafe.getShort(baseObject, address + size);
+ short aShort = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size);
byte[] col = new byte[aShort];
size += 2;
- CarbonUnsafe.unsafe
+ CarbonUnsafe.getUnsafe()
.copyMemory(baseObject, address + size, col, CarbonUnsafe.BYTE_ARRAY_OFFSET,
col.length);
size += col.length;
rowToFill[dimCount] = col;
} else {
- int anInt = CarbonUnsafe.unsafe.getInt(baseObject, address + size);
+ int anInt = CarbonUnsafe.getUnsafe().getInt(baseObject, address + size);
size += 4;
rowToFill[dimCount] = anInt;
}
@@ -192,10 +192,10 @@ public class UnsafeCarbonRowPage {
// write complex dimensions here.
for (; dimCount < dimensionSize; dimCount++) {
- short aShort = CarbonUnsafe.unsafe.getShort(baseObject, address + size);
+ short aShort = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size);
byte[] col = new byte[aShort];
size += 2;
- CarbonUnsafe.unsafe
+ CarbonUnsafe.getUnsafe()
.copyMemory(baseObject, address + size, col, CarbonUnsafe.BYTE_ARRAY_OFFSET, col.length);
size += col.length;
rowToFill[dimCount] = col;
@@ -203,7 +203,7 @@ public class UnsafeCarbonRowPage {
int nullSetSize = nullSetWords.length * 8;
Arrays.fill(nullSetWords, 0);
- CarbonUnsafe.unsafe
+ CarbonUnsafe.getUnsafe()
.copyMemory(baseObject, address + size, nullSetWords, CarbonUnsafe.LONG_ARRAY_OFFSET,
nullSetSize);
size += nullSetSize;
@@ -212,30 +212,30 @@ public class UnsafeCarbonRowPage {
if (isSet(nullSetWords, mesCount)) {
switch (measureDataType[mesCount]) {
case SHORT:
- Short sval = CarbonUnsafe.unsafe.getShort(baseObject, address + size);
+ Short sval = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size);
size += 2;
rowToFill[dimensionSize + mesCount] = sval;
break;
case INT:
- Integer ival = CarbonUnsafe.unsafe.getInt(baseObject, address + size);
+ Integer ival = CarbonUnsafe.getUnsafe().getInt(baseObject, address + size);
size += 4;
rowToFill[dimensionSize + mesCount] = ival;
break;
case LONG:
- Long val = CarbonUnsafe.unsafe.getLong(baseObject, address + size);
+ Long val = CarbonUnsafe.getUnsafe().getLong(baseObject, address + size);
size += 8;
rowToFill[dimensionSize + mesCount] = val;
break;
case DOUBLE:
- Double doubleVal = CarbonUnsafe.unsafe.getDouble(baseObject, address + size);
+ Double doubleVal = CarbonUnsafe.getUnsafe().getDouble(baseObject, address + size);
size += 8;
rowToFill[dimensionSize + mesCount] = doubleVal;
break;
case DECIMAL:
- short aShort = CarbonUnsafe.unsafe.getShort(baseObject, address + size);
+ short aShort = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size);
byte[] bigDecimalInBytes = new byte[aShort];
size += 2;
- CarbonUnsafe.unsafe.copyMemory(baseObject, address + size, bigDecimalInBytes,
+ CarbonUnsafe.getUnsafe().copyMemory(baseObject, address + size, bigDecimalInBytes,
CarbonUnsafe.BYTE_ARRAY_OFFSET, bigDecimalInBytes.length);
size += bigDecimalInBytes.length;
rowToFill[dimensionSize + mesCount] = DataTypeUtil.byteToBigDecimal(bigDecimalInBytes);
@@ -255,17 +255,17 @@ public class UnsafeCarbonRowPage {
Object baseObject = dataBlock.getBaseObject();
for (; dimCount < noDictionaryDimensionMapping.length; dimCount++) {
if (noDictionaryDimensionMapping[dimCount]) {
- short aShort = CarbonUnsafe.unsafe.getShort(baseObject, address + size);
+ short aShort = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size);
byte[] col = new byte[aShort];
size += 2;
- CarbonUnsafe.unsafe
+ CarbonUnsafe.getUnsafe()
.copyMemory(baseObject, address + size, col, CarbonUnsafe.BYTE_ARRAY_OFFSET,
col.length);
size += col.length;
stream.writeShort(aShort);
stream.write(col);
} else {
- int anInt = CarbonUnsafe.unsafe.getInt(baseObject, address + size);
+ int anInt = CarbonUnsafe.getUnsafe().getInt(baseObject, address + size);
size += 4;
stream.writeInt(anInt);
}
@@ -273,10 +273,10 @@ public class UnsafeCarbonRowPage {
// write complex dimensions here.
for (; dimCount < dimensionSize; dimCount++) {
- short aShort = CarbonUnsafe.unsafe.getShort(baseObject, address + size);
+ short aShort = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size);
byte[] col = new byte[aShort];
size += 2;
- CarbonUnsafe.unsafe
+ CarbonUnsafe.getUnsafe()
.copyMemory(baseObject, address + size, col, CarbonUnsafe.BYTE_ARRAY_OFFSET, col.length);
size += col.length;
stream.writeShort(aShort);
@@ -285,7 +285,7 @@ public class UnsafeCarbonRowPage {
int nullSetSize = nullSetWords.length * 8;
Arrays.fill(nullSetWords, 0);
- CarbonUnsafe.unsafe
+ CarbonUnsafe.getUnsafe()
.copyMemory(baseObject, address + size, nullSetWords, CarbonUnsafe.LONG_ARRAY_OFFSET,
nullSetSize);
size += nullSetSize;
@@ -297,30 +297,30 @@ public class UnsafeCarbonRowPage {
if (isSet(nullSetWords, mesCount)) {
switch (measureDataType[mesCount]) {
case SHORT:
- short sval = CarbonUnsafe.unsafe.getShort(baseObject, address + size);
+ short sval = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size);
size += 2;
stream.writeShort(sval);
break;
case INT:
- int ival = CarbonUnsafe.unsafe.getInt(baseObject, address + size);
+ int ival = CarbonUnsafe.getUnsafe().getInt(baseObject, address + size);
size += 4;
stream.writeInt(ival);
break;
case LONG:
- long val = CarbonUnsafe.unsafe.getLong(baseObject, address + size);
+ long val = CarbonUnsafe.getUnsafe().getLong(baseObject, address + size);
size += 8;
stream.writeLong(val);
break;
case DOUBLE:
- double doubleVal = CarbonUnsafe.unsafe.getDouble(baseObject, address + size);
+ double doubleVal = CarbonUnsafe.getUnsafe().getDouble(baseObject, address + size);
size += 8;
stream.writeDouble(doubleVal);
break;
case DECIMAL:
- short aShort = CarbonUnsafe.unsafe.getShort(baseObject, address + size);
+ short aShort = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size);
byte[] bigDecimalInBytes = new byte[aShort];
size += 2;
- CarbonUnsafe.unsafe.copyMemory(baseObject, address + size, bigDecimalInBytes,
+ CarbonUnsafe.getUnsafe().copyMemory(baseObject, address + size, bigDecimalInBytes,
CarbonUnsafe.BYTE_ARRAY_OFFSET, bigDecimalInBytes.length);
size += bigDecimalInBytes.length;
stream.writeShort(aShort);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeSortDataRows.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeSortDataRows.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeSortDataRows.java
index fd0bec4..d0bb4f6 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeSortDataRows.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeSortDataRows.java
@@ -383,7 +383,7 @@ public class UnsafeSortDataRows {
MemoryBlock newMemoryBlock = UnsafeSortMemoryManager.INSTANCE
.allocateMemoryLazy(taskId, page.getDataBlock().size());
// copying data from working memory manager to sortmemory manager
- CarbonUnsafe.unsafe
+ CarbonUnsafe.getUnsafe()
.copyMemory(page.getDataBlock().getBaseObject(), page.getDataBlock().getBaseOffset(),
newMemoryBlock.getBaseObject(), newMemoryBlock.getBaseOffset(),
page.getDataBlock().size());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/comparator/UnsafeRowComparator.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/comparator/UnsafeRowComparator.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/comparator/UnsafeRowComparator.java
index 1d60b11..c54dcd6 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/comparator/UnsafeRowComparator.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/comparator/UnsafeRowComparator.java
@@ -49,17 +49,17 @@ public class UnsafeRowComparator implements Comparator<UnsafeCarbonRow> {
int sizeB = 0;
for (boolean isNoDictionary : noDictionarySortColumnMaping) {
if (isNoDictionary) {
- short aShort1 = CarbonUnsafe.unsafe.getShort(baseObject, rowA + sizeA);
+ short aShort1 = CarbonUnsafe.getUnsafe().getShort(baseObject, rowA + sizeA);
byte[] byteArr1 = new byte[aShort1];
sizeA += 2;
- CarbonUnsafe.unsafe.copyMemory(baseObject, rowA + sizeA, byteArr1,
+ CarbonUnsafe.getUnsafe().copyMemory(baseObject, rowA + sizeA, byteArr1,
CarbonUnsafe.BYTE_ARRAY_OFFSET, aShort1);
sizeA += aShort1;
- short aShort2 = CarbonUnsafe.unsafe.getShort(baseObject, rowB + sizeB);
+ short aShort2 = CarbonUnsafe.getUnsafe().getShort(baseObject, rowB + sizeB);
byte[] byteArr2 = new byte[aShort2];
sizeB += 2;
- CarbonUnsafe.unsafe.copyMemory(baseObject, rowB + sizeB, byteArr2,
+ CarbonUnsafe.getUnsafe().copyMemory(baseObject, rowB + sizeB, byteArr2,
CarbonUnsafe.BYTE_ARRAY_OFFSET, aShort2);
sizeB += aShort2;
@@ -68,9 +68,9 @@ public class UnsafeRowComparator implements Comparator<UnsafeCarbonRow> {
return difference;
}
} else {
- int dimFieldA = CarbonUnsafe.unsafe.getInt(baseObject, rowA + sizeA);
+ int dimFieldA = CarbonUnsafe.getUnsafe().getInt(baseObject, rowA + sizeA);
sizeA += 4;
- int dimFieldB = CarbonUnsafe.unsafe.getInt(baseObject, rowB + sizeB);
+ int dimFieldB = CarbonUnsafe.getUnsafe().getInt(baseObject, rowB + sizeB);
sizeB += 4;
diff = dimFieldA - dimFieldB;
if (diff != 0) {
@@ -94,18 +94,18 @@ public class UnsafeRowComparator implements Comparator<UnsafeCarbonRow> {
int sizeB = 0;
for (boolean isNoDictionary : noDictionarySortColumnMaping) {
if (isNoDictionary) {
- short aShort1 = CarbonUnsafe.unsafe.getShort(baseObjectL, rowA + sizeA);
+ short aShort1 = CarbonUnsafe.getUnsafe().getShort(baseObjectL, rowA + sizeA);
byte[] byteArr1 = new byte[aShort1];
sizeA += 2;
- CarbonUnsafe.unsafe
+ CarbonUnsafe.getUnsafe()
.copyMemory(baseObjectL, rowA + sizeA, byteArr1, CarbonUnsafe.BYTE_ARRAY_OFFSET,
aShort1);
sizeA += aShort1;
- short aShort2 = CarbonUnsafe.unsafe.getShort(baseObjectR, rowB + sizeB);
+ short aShort2 = CarbonUnsafe.getUnsafe().getShort(baseObjectR, rowB + sizeB);
byte[] byteArr2 = new byte[aShort2];
sizeB += 2;
- CarbonUnsafe.unsafe
+ CarbonUnsafe.getUnsafe()
.copyMemory(baseObjectR, rowB + sizeB, byteArr2, CarbonUnsafe.BYTE_ARRAY_OFFSET,
aShort2);
sizeB += aShort2;
@@ -115,9 +115,9 @@ public class UnsafeRowComparator implements Comparator<UnsafeCarbonRow> {
return difference;
}
} else {
- int dimFieldA = CarbonUnsafe.unsafe.getInt(baseObjectL, rowA + sizeA);
+ int dimFieldA = CarbonUnsafe.getUnsafe().getInt(baseObjectL, rowA + sizeA);
sizeA += 4;
- int dimFieldB = CarbonUnsafe.unsafe.getInt(baseObjectR, rowB + sizeB);
+ int dimFieldB = CarbonUnsafe.getUnsafe().getInt(baseObjectR, rowB + sizeB);
sizeB += 4;
diff = dimFieldA - dimFieldB;
if (diff != 0) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/comparator/UnsafeRowComparatorForNormalDIms.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/comparator/UnsafeRowComparatorForNormalDIms.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/comparator/UnsafeRowComparatorForNormalDIms.java
index 4fd245f..53f976f 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/comparator/UnsafeRowComparatorForNormalDIms.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/comparator/UnsafeRowComparatorForNormalDIms.java
@@ -44,9 +44,9 @@ public class UnsafeRowComparatorForNormalDIms implements Comparator<UnsafeCarbon
int sizeA = 0;
int sizeB = 0;
for (int i = 0; i < numberOfSortColumns; i++) {
- int dimFieldA = CarbonUnsafe.unsafe.getInt(baseObject, rowA + sizeA);
+ int dimFieldA = CarbonUnsafe.getUnsafe().getInt(baseObject, rowA + sizeA);
sizeA += 4;
- int dimFieldB = CarbonUnsafe.unsafe.getInt(baseObject, rowB + sizeB);
+ int dimFieldB = CarbonUnsafe.getUnsafe().getInt(baseObject, rowB + sizeB);
sizeB += 4;
diff = dimFieldA - dimFieldB;
if (diff != 0) {
[3/3] carbondata git commit: [Review][CARBONDATA-1386] fixed findbugs
errors in carbondata-core
Posted by ja...@apache.org.
[Review][CARBONDATA-1386] fixed findbugs errors in carbondata-core
fixed findbugs errors in carbondata-core
This closes #1263
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/500654e6
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/500654e6
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/500654e6
Branch: refs/heads/master
Commit: 500654e60d4fb7b1ab5eda39f29a415933838006
Parents: 2ee7775
Author: sraghunandan <ca...@gmail.com>
Authored: Thu Aug 17 19:14:48 2017 +0530
Committer: Jacky Li <ja...@qq.com>
Committed: Sat Aug 19 09:52:52 2017 +0800
----------------------------------------------------------------------
.../impl/ExtendedRollingFileAppender.java | 4 +
.../dictionary/AbstractDictionaryCache.java | 2 +-
.../dictionary/DoubleArrayTrieDictionary.java | 12 +-
.../dictionary/ReverseDictionaryCache.java | 2 +-
.../core/constants/CarbonCommonConstants.java | 3 +-
.../core/datamap/DataMapStoreManager.java | 4 +-
.../carbondata/core/datamap/TableDataMap.java | 6 +-
.../datastore/AbstractBlockIndexStoreCache.java | 8 +-
.../core/datastore/BlockIndexStore.java | 2 +-
.../core/datastore/block/TableBlockInfo.java | 12 +-
.../core/datastore/block/TableTaskInfo.java | 23 ++++
.../impl/FixedLengthDimensionDataChunk.java | 8 ++
.../UnsafeAbstractDimensionDataChunkStore.java | 10 +-
...nsafeFixedLengthDimensionDataChunkStore.java | 24 ++--
...afeVariableLengthDimesionDataChunkStore.java | 28 ++---
.../columnar/BlockIndexerStorageForShort.java | 2 +-
.../core/datastore/impl/FileFactory.java | 3 +
.../core/datastore/impl/FileHolderImpl.java | 8 +-
.../core/datastore/page/ColumnPage.java | 2 +
.../page/UnsafeFixLengthColumnPage.java | 87 +++++++------
.../page/UnsafeVarLengthColumnPage.java | 10 +-
.../page/encoding/DeltaIntegralCodec.java | 9 +-
.../core/datastore/page/encoding/RLECodec.java | 2 +
.../statistics/PrimitivePageStatsCollector.java | 13 ++
.../core/datastore/row/LoadStatusType.java | 4 -
.../client/DictionaryClientHandler.java | 2 +-
.../generator/ServerDictionaryGenerator.java | 11 +-
.../generator/TableDictionaryGenerator.java | 35 +++---
.../dictionary/server/DictionaryServer.java | 2 +-
.../carbondata/core/indexstore/Blocklet.java | 11 +-
.../core/indexstore/UnsafeMemoryDMStore.java | 39 +++---
.../blockletindex/BlockletDataMap.java | 19 +--
.../BlockletDataRefNodeWrapper.java | 2 +-
.../core/indexstore/row/UnsafeDataMapRow.java | 25 ++--
.../MultiDimKeyVarLengthEquiSplitGenerator.java | 23 ----
...tiDimKeyVarLengthVariableSplitGenerator.java | 23 ----
.../DateDirectDictionaryGenerator.java | 2 +-
.../TimeStampDirectDictionaryGenerator.java | 2 +-
.../carbondata/core/locks/HdfsFileLock.java | 2 +-
.../carbondata/core/locks/ZooKeeperLocking.java | 12 +-
.../carbondata/core/memory/CarbonUnsafe.java | 6 +-
.../core/memory/IntPointerBuffer.java | 4 +-
.../carbondata/core/memory/MemoryBlock.java | 7 --
.../core/memory/UnsafeMemoryAllocator.java | 6 +-
.../core/memory/UnsafeMemoryManager.java | 12 +-
.../core/memory/UnsafeSortMemoryManager.java | 12 +-
.../core/metadata/ColumnPageCodecMeta.java | 2 +
.../datatype/DecimalConverterFactory.java | 6 -
.../core/metadata/schema/table/TableInfo.java | 10 +-
.../core/mutate/CarbonUpdateUtil.java | 4 -
.../RestructureBasedRawResultCollector.java | 3 +-
.../scan/complextypes/ComplexQueryType.java | 3 -
.../core/scan/executor/util/QueryUtil.java | 3 +-
.../scan/executor/util/RestructureUtil.java | 4 +-
.../expression/RangeExpressionEvaluator.java | 20 +--
.../expression/conditional/ListExpression.java | 11 +-
.../scan/filter/FilterExpressionProcessor.java | 15 ++-
.../carbondata/core/scan/filter/FilterUtil.java | 8 +-
.../RestructureExcludeFilterExecutorImpl.java | 12 +-
.../RestructureIncludeFilterExecutorImpl.java | 13 +-
.../executer/RowLevelFilterExecuterImpl.java | 4 +-
.../scan/filter/intf/FilterOptimizerImpl.java | 4 +-
.../filter/optimizer/RangeFilterOptmizer.java | 2 +-
.../MeasureColumnResolvedFilterInfo.java | 6 -
.../visitor/CustomTypeDictionaryVisitor.java | 39 +++---
.../visitor/DictionaryColumnVisitor.java | 48 +++----
.../visitor/MeasureColumnVisitor.java | 50 ++++----
.../visitor/NoDictionaryTypeVisitor.java | 48 +++----
.../visitor/RangeDictionaryColumnVisitor.java | 28 +++--
.../visitor/RangeDirectDictionaryVisitor.java | 53 ++++----
.../visitor/RangeNoDictionaryTypeVisitor.java | 52 ++++----
.../scan/result/vector/ColumnVectorInfo.java | 23 ++++
.../core/stats/QueryStatisticsRecorderImpl.java | 14 ++-
.../apache/carbondata/core/util/ByteUtil.java | 49 ++++----
.../carbondata/core/util/CarbonProperties.java | 28 ++---
.../apache/carbondata/core/util/CarbonUtil.java | 11 +-
.../core/util/DataTypeConverterImpl.java | 2 +-
.../carbondata/core/util/DataTypeUtil.java | 6 +-
.../core/util/comparator/Comparator.java | 8 +-
dev/findbugs-exclude.xml | 124 ++++---------------
pom.xml | 2 +-
.../sort/unsafe/UnsafeCarbonRowPage.java | 72 +++++------
.../newflow/sort/unsafe/UnsafeSortDataRows.java | 2 +-
.../unsafe/comparator/UnsafeRowComparator.java | 24 ++--
.../UnsafeRowComparatorForNormalDIms.java | 4 +-
85 files changed, 663 insertions(+), 699 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/common/src/main/java/org/apache/carbondata/common/logging/impl/ExtendedRollingFileAppender.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/carbondata/common/logging/impl/ExtendedRollingFileAppender.java b/common/src/main/java/org/apache/carbondata/common/logging/impl/ExtendedRollingFileAppender.java
index 1261726..828dd14 100644
--- a/common/src/main/java/org/apache/carbondata/common/logging/impl/ExtendedRollingFileAppender.java
+++ b/common/src/main/java/org/apache/carbondata/common/logging/impl/ExtendedRollingFileAppender.java
@@ -74,6 +74,10 @@ public class ExtendedRollingFileAppender extends RollingFileAppender {
}
});
+ if (null == files) {
+ return;
+ }
+
int backupFiles = files.length - 1;
if (backupFiles <= maxBackupIndex) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java
index f0b8c78..6910888 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java
@@ -173,7 +173,7 @@ public abstract class AbstractDictionaryCache<K extends DictionaryColumnUniqueId
protected long getSortIndexSize(long numOfRecords) {
// sort index has sort index and reverse sort index,each is 4 byte integer.
// 32 byte is the array header of both the integer arrays
- return numOfRecords * ObjectSizeCalculator.estimate(new Integer(0), 16) * 2 + 32;
+ return numOfRecords * ObjectSizeCalculator.estimate(0, 16) * 2 + 32;
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java
index cab1926..ef36d7a 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java
@@ -35,10 +35,10 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
*/
public class DoubleArrayTrieDictionary {
- public static final byte[] HEAD_MAGIC = new byte[]{
+ private static final byte[] HEAD_MAGIC = new byte[]{
0x44, 0x41, 0x54, 0x54, 0x72, 0x69, 0x65, 0x44, 0x69, 0x63, 0x74
}; // "DATTrieDict"
- public static final int HEAD_LEN = HEAD_MAGIC.length;
+ private static final int HEAD_LEN = HEAD_MAGIC.length;
private static final int INIT_CAPA_VALUE = 256; // init len of double array
private static final int BASE_ROOT_VALUE = 1; // root base value of trie root
@@ -168,7 +168,7 @@ public class DoubleArrayTrieDictionary {
return null;
}
if (check[cpos] == pos) {
- children.add(new Integer(i));
+ children.add(i);
}
}
return children;
@@ -227,9 +227,9 @@ public class DoubleArrayTrieDictionary {
private int conflict(int start, int bKey) {
int from = start;
TreeSet<Integer> children = getChildren(from);
- children.add(new Integer(bKey));
+ children.add(bKey);
int newBasePos = findFreeRoom(children);
- children.remove(new Integer(bKey));
+ children.remove(bKey);
int oldBasePos = base[start];
base[start] = newBasePos;
@@ -351,7 +351,7 @@ public class DoubleArrayTrieDictionary {
*/
public void read(DataInputStream in) throws IOException {
byte[] header = new byte[HEAD_LEN];
- in.read(header);
+ in.readFully(header);
int comp = 0;
for (int i = 0; i < HEAD_LEN; i++) {
comp = HEAD_MAGIC[i] - header[i];
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCache.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCache.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCache.java
index 28568b5..fd5a954 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCache.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCache.java
@@ -58,7 +58,7 @@ public class ReverseDictionaryCache<K extends DictionaryColumnUniqueIdentifier,
private static final long sizeOfHashMapNode = ObjectSizeCalculator.estimate(new
DictionaryByteArrayWrapper(new byte[0]), 16) +
- ObjectSizeCalculator.estimate(new Integer(0), 16);
+ ObjectSizeCalculator.estimate(0, 16);
private static final long byteArraySize = ObjectSizeCalculator.estimate(new byte[0], 16);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 8939a7e..944abc4 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -189,8 +189,7 @@ public final class CarbonCommonConstants {
/**
* Bytes for string 0, it is used in codegen in case of null values.
*/
- public static final byte[] ZERO_BYTE_ARRAY = "0".getBytes();
-
+ public static final byte[] ZERO_BYTE_ARRAY = "0".getBytes(Charset.forName(DEFAULT_CHARSET));
/**
* FILE STATUS IN-PROGRESS
*/
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
index f5bc22f..54318b5 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
@@ -92,7 +92,7 @@ public final class DataMapStoreManager {
try {
DataMapFactory dataMapFactory = factoryClass.newInstance();
dataMapFactory.init(identifier, dataMapName);
- dataMap = new TableDataMap(identifier, dataMapName, dataMapFactory);
+ dataMap = new TableDataMap(dataMapName, dataMapFactory);
} catch (Exception e) {
LOGGER.error(e);
throw new RuntimeException(e);
@@ -119,7 +119,7 @@ public final class DataMapStoreManager {
* @param dataMapName
*/
public void clearDataMap(AbsoluteTableIdentifier identifier, String dataMapName) {
- List<TableDataMap> tableDataMaps = allDataMaps.get(identifier);
+ List<TableDataMap> tableDataMaps = allDataMaps.get(identifier.uniqueName());
if (tableDataMaps != null) {
int i = 0;
for (TableDataMap tableDataMap: tableDataMaps) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
index b55c5d9..5571538 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
@@ -25,7 +25,6 @@ import org.apache.carbondata.core.datamap.dev.DataMapFactory;
import org.apache.carbondata.core.events.ChangeEvent;
import org.apache.carbondata.core.events.EventListener;
import org.apache.carbondata.core.indexstore.Blocklet;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
/**
@@ -34,8 +33,6 @@ import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
*/
public final class TableDataMap implements EventListener {
- private AbsoluteTableIdentifier identifier;
-
private String dataMapName;
private DataMapFactory dataMapFactory;
@@ -43,9 +40,8 @@ public final class TableDataMap implements EventListener {
/**
* It is called to initialize and load the required table datamap metadata.
*/
- public TableDataMap(AbsoluteTableIdentifier identifier, String dataMapName,
+ public TableDataMap(String dataMapName,
DataMapFactory dataMapFactory) {
- this.identifier = identifier;
this.dataMapName = dataMapName;
this.dataMapFactory = dataMapFactory;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/AbstractBlockIndexStoreCache.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/AbstractBlockIndexStoreCache.java b/core/src/main/java/org/apache/carbondata/core/datastore/AbstractBlockIndexStoreCache.java
index 8cfc602..3a62783 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/AbstractBlockIndexStoreCache.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/AbstractBlockIndexStoreCache.java
@@ -41,10 +41,7 @@ import org.apache.carbondata.core.util.ObjectSizeCalculator;
*/
public abstract class AbstractBlockIndexStoreCache<K, V>
implements Cache<TableBlockUniqueIdentifier, AbstractIndex> {
- /**
- * carbon store path
- */
- protected String carbonStorePath;
+
/**
* CarbonLRU cache
*/
@@ -70,8 +67,7 @@ public abstract class AbstractBlockIndexStoreCache<K, V>
*/
protected Map<String, Object> segmentIDLock;
- public AbstractBlockIndexStoreCache(String carbonStorePath, CarbonLRUCache lruCache) {
- this.carbonStorePath = carbonStorePath;
+ public AbstractBlockIndexStoreCache(CarbonLRUCache lruCache) {
this.lruCache = lruCache;
blockInfoLock = new ConcurrentHashMap<BlockInfo, Object>();
segmentIDLock = new ConcurrentHashMap<String, Object>();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/BlockIndexStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/BlockIndexStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/BlockIndexStore.java
index 32ca096..fad6ed6 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/BlockIndexStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/BlockIndexStore.java
@@ -59,7 +59,7 @@ public class BlockIndexStore<K, V> extends AbstractBlockIndexStoreCache<K, V> {
private static final LogService LOGGER =
LogServiceFactory.getLogService(BlockIndexStore.class.getName());
public BlockIndexStore(String carbonStorePath, CarbonLRUCache lruCache) {
- super(carbonStorePath, lruCache);
+ super(lruCache);
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
index 316e202..6389528 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
@@ -190,13 +190,15 @@ public class TableBlockInfo implements Distributable, Serializable {
if (blockLength != other.blockLength) {
return false;
}
- if (filePath == null && other.filePath != null) {
- return false;
- } else if (filePath != null && other.filePath == null) {
- return false;
- } else if (!filePath.equals(other.filePath)) {
+
+ if (null == filePath || null == other.filePath) {
+ return false;
+ }
+
+ if (!filePath.equals(other.filePath)) {
return false;
}
+
if (blockletInfos.getStartBlockletNumber() != other.blockletInfos.getStartBlockletNumber()) {
return false;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/block/TableTaskInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableTaskInfo.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableTaskInfo.java
index 73aa810..89a7551 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableTaskInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableTaskInfo.java
@@ -60,6 +60,29 @@ public class TableTaskInfo implements Distributable {
return taskId.compareTo(((TableTaskInfo)o).getTaskId());
}
+ @Override public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+
+ if (null == obj) {
+ return false;
+ }
+
+ if (!(obj instanceof TableTaskInfo)) {
+ return false;
+ }
+
+ return 0 == taskId.compareTo(((TableTaskInfo)obj).getTaskId());
+ }
+
+ @Override public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((taskId == null) ? 0 : taskId.hashCode());
+ return result;
+ }
+
/**
* Finding which node has the maximum number of blocks for it.
* @param blockList
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
index bb76bbf..97fd226 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
@@ -110,6 +110,10 @@ public class FixedLengthDimensionDataChunk extends AbstractDimensionDataChunk {
case LONG:
vector.putLong(vectorOffset++, (long) valueFromSurrogate);
break;
+ default:
+ throw new IllegalArgumentException(
+ "unsupported data type: " + columnVectorInfo.directDictionaryGenerator
+ .getReturnType());
}
}
}
@@ -150,6 +154,10 @@ public class FixedLengthDimensionDataChunk extends AbstractDimensionDataChunk {
case LONG:
vector.putLong(vectorOffset++, (long) valueFromSurrogate);
break;
+ default:
+ throw new IllegalArgumentException(
+ "unsupported data type: " + columnVectorInfo.directDictionaryGenerator
+ .getReturnType());
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java
index 3752fb6..704f2d3 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java
@@ -92,16 +92,16 @@ public abstract class UnsafeAbstractDimensionDataChunkStore implements Dimension
invertedIndex.length * CarbonCommonConstants.INT_SIZE_IN_BYTE;
}
// copy the data to memory
- CarbonUnsafe.unsafe
+ CarbonUnsafe.getUnsafe()
.copyMemory(data, CarbonUnsafe.BYTE_ARRAY_OFFSET, dataPageMemoryBlock.getBaseObject(),
dataPageMemoryBlock.getBaseOffset(), this.dataLength);
// if inverted index is present then copy the inverted index
// and reverse inverted index to memory
if (isExplicitSorted) {
- CarbonUnsafe.unsafe.copyMemory(invertedIndex, CarbonUnsafe.INT_ARRAY_OFFSET,
+ CarbonUnsafe.getUnsafe().copyMemory(invertedIndex, CarbonUnsafe.INT_ARRAY_OFFSET,
dataPageMemoryBlock.getBaseObject(), dataPageMemoryBlock.getBaseOffset() + dataLength,
invertedIndex.length * CarbonCommonConstants.INT_SIZE_IN_BYTE);
- CarbonUnsafe.unsafe.copyMemory(invertedIndexReverse, CarbonUnsafe.INT_ARRAY_OFFSET,
+ CarbonUnsafe.getUnsafe().copyMemory(invertedIndexReverse, CarbonUnsafe.INT_ARRAY_OFFSET,
dataPageMemoryBlock.getBaseObject(),
dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset,
invertedIndexReverse.length * CarbonCommonConstants.INT_SIZE_IN_BYTE);
@@ -129,8 +129,8 @@ public abstract class UnsafeAbstractDimensionDataChunkStore implements Dimension
* @return inverted index based on row id passed
*/
@Override public int getInvertedIndex(int rowId) {
- return CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
- dataPageMemoryBlock.getBaseOffset() + dataLength + (rowId
+ return CarbonUnsafe.getUnsafe().getInt(dataPageMemoryBlock.getBaseObject(),
+ dataPageMemoryBlock.getBaseOffset() + dataLength + ((long)rowId
* CarbonCommonConstants.INT_SIZE_IN_BYTE));
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java
index 4ff9c77..8c8d08f 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java
@@ -53,16 +53,16 @@ public class UnsafeFixedLengthDimensionDataChunkStore
@Override public byte[] getRow(int rowId) {
// if column was explicitly sorted we need to get the rowid based inverted index reverse
if (isExplicitSorted) {
- rowId = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
- dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset + (rowId
+ rowId = CarbonUnsafe.getUnsafe().getInt(dataPageMemoryBlock.getBaseObject(),
+ dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset + ((long)rowId
* CarbonCommonConstants.INT_SIZE_IN_BYTE));
}
// creating a row
byte[] data = new byte[columnValueSize];
//copy the row from memory block based on offset
// offset position will be index * each column value length
- CarbonUnsafe.unsafe.copyMemory(dataPageMemoryBlock.getBaseObject(),
- dataPageMemoryBlock.getBaseOffset() + (rowId * columnValueSize), data,
+ CarbonUnsafe.getUnsafe().copyMemory(dataPageMemoryBlock.getBaseObject(),
+ dataPageMemoryBlock.getBaseOffset() + ((long)rowId * columnValueSize), data,
CarbonUnsafe.BYTE_ARRAY_OFFSET, columnValueSize);
return data;
}
@@ -77,8 +77,8 @@ public class UnsafeFixedLengthDimensionDataChunkStore
@Override public int getSurrogate(int index) {
// if column was explicitly sorted we need to get the rowid based inverted index reverse
if (isExplicitSorted) {
- index = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
- dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset + (index
+ index = CarbonUnsafe.getUnsafe().getInt(dataPageMemoryBlock.getBaseObject(),
+ dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset + ((long)index
* CarbonCommonConstants.INT_SIZE_IN_BYTE));
}
// below part is to convert the byte array to surrogate value
@@ -86,7 +86,7 @@ public class UnsafeFixedLengthDimensionDataChunkStore
int surrogate = 0;
for (int i = 0; i < columnValueSize; i++) {
surrogate <<= 8;
- surrogate ^= CarbonUnsafe.unsafe.getByte(dataPageMemoryBlock.getBaseObject(),
+ surrogate ^= CarbonUnsafe.getUnsafe().getByte(dataPageMemoryBlock.getBaseObject(),
dataPageMemoryBlock.getBaseOffset() + startOffsetOfData) & 0xFF;
startOffsetOfData++;
}
@@ -103,14 +103,14 @@ public class UnsafeFixedLengthDimensionDataChunkStore
@Override public void fillRow(int rowId, byte[] buffer, int offset) {
// if column was explicitly sorted we need to get the rowid based inverted index reverse
if (isExplicitSorted) {
- rowId = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
- dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset + (rowId
+ rowId = CarbonUnsafe.getUnsafe().getInt(dataPageMemoryBlock.getBaseObject(),
+ dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset + ((long)rowId
* CarbonCommonConstants.INT_SIZE_IN_BYTE));
}
//copy the row from memory block based on offset
// offset position will be index * each column value length
- CarbonUnsafe.unsafe.copyMemory(dataPageMemoryBlock.getBaseObject(),
- dataPageMemoryBlock.getBaseOffset() + (rowId * columnValueSize), buffer,
+ CarbonUnsafe.getUnsafe().copyMemory(dataPageMemoryBlock.getBaseObject(),
+ dataPageMemoryBlock.getBaseOffset() + ((long)rowId * columnValueSize), buffer,
CarbonUnsafe.BYTE_ARRAY_OFFSET + offset, columnValueSize);
}
@@ -133,7 +133,7 @@ public class UnsafeFixedLengthDimensionDataChunkStore
index = index * columnValueSize;
int compareResult = 0;
for (int i = 0; i < compareValue.length; i++) {
- compareResult = (CarbonUnsafe.unsafe
+ compareResult = (CarbonUnsafe.getUnsafe()
.getByte(dataPageMemoryBlock.getBaseObject(), dataPageMemoryBlock.getBaseOffset() + index)
& 0xff) - (compareValue[i] & 0xff);
if (compareResult != 0) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
index c6c98f9..6193804 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
@@ -68,7 +68,7 @@ public class UnsafeVariableLengthDimesionDataChunkStore
// position from where offsets will start
this.dataPointersOffsets = this.invertedIndexReverseOffset;
if (isExplicitSorted) {
- this.dataPointersOffsets += numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE;
+ this.dataPointersOffsets += (long)numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE;
}
// As data is of variable length and data format is
// <length in short><data><length in short><data>
@@ -87,7 +87,7 @@ public class UnsafeVariableLengthDimesionDataChunkStore
// as first position will be start from 2 byte as data is stored first in the memory block
// we need to skip first two bytes this is because first two bytes will be length of the data
// which we have to skip
- CarbonUnsafe.unsafe.putInt(dataPageMemoryBlock.getBaseObject(),
+ CarbonUnsafe.getUnsafe().putInt(dataPageMemoryBlock.getBaseObject(),
dataPageMemoryBlock.getBaseOffset() + pointerOffsets,
CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
// incrementing the pointers as first value is already filled and as we are storing as int
@@ -101,7 +101,7 @@ public class UnsafeVariableLengthDimesionDataChunkStore
// as first offset is already stored, we need to start from the 2nd row in data array
for (int i = 1; i < numberOfRows; i++) {
// first copy the length of previous row
- CarbonUnsafe.unsafe.copyMemory(dataPageMemoryBlock.getBaseObject(),
+ CarbonUnsafe.getUnsafe().copyMemory(dataPageMemoryBlock.getBaseObject(),
dataPageMemoryBlock.getBaseOffset() + startOffset, length, CarbonUnsafe.BYTE_ARRAY_OFFSET,
CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
buffer.put(length);
@@ -114,7 +114,7 @@ public class UnsafeVariableLengthDimesionDataChunkStore
buffer.clear();
// now put the offset of current row, here we need to add 2 more bytes as current will
// also have length part so we have to skip length
- CarbonUnsafe.unsafe.putInt(dataPageMemoryBlock.getBaseObject(),
+ CarbonUnsafe.getUnsafe().putInt(dataPageMemoryBlock.getBaseObject(),
dataPageMemoryBlock.getBaseOffset() + pointerOffsets,
startOffset + CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
// incrementing the pointers as first value is already filled and as we are storing as int
@@ -133,8 +133,8 @@ public class UnsafeVariableLengthDimesionDataChunkStore
@Override public byte[] getRow(int rowId) {
// if column was explicitly sorted we need to get the rowid based inverted index reverse
if (isExplicitSorted) {
- rowId = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
- dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset + (rowId
+ rowId = CarbonUnsafe.getUnsafe().getInt(dataPageMemoryBlock.getBaseObject(),
+ dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset + ((long)rowId
* CarbonCommonConstants.INT_SIZE_IN_BYTE));
}
// now to get the row from memory block we need to do following thing
@@ -143,13 +143,13 @@ public class UnsafeVariableLengthDimesionDataChunkStore
// Subtract the current row offset + 2 bytes(to skip the data length) with next row offset
// else subtract the current row offset + 2 bytes(to skip the data length)
// with complete data length
- int currentDataOffset = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
+ int currentDataOffset = CarbonUnsafe.getUnsafe().getInt(dataPageMemoryBlock.getBaseObject(),
dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets + (rowId
* CarbonCommonConstants.INT_SIZE_IN_BYTE));
short length = 0;
// calculating the length of data
if (rowId < numberOfRows - 1) {
- int OffsetOfNextdata = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
+ int OffsetOfNextdata = CarbonUnsafe.getUnsafe().getInt(dataPageMemoryBlock.getBaseObject(),
dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets + ((rowId + 1)
* CarbonCommonConstants.INT_SIZE_IN_BYTE));
length = (short) (OffsetOfNextdata - (currentDataOffset
@@ -159,7 +159,7 @@ public class UnsafeVariableLengthDimesionDataChunkStore
length = (short) (this.dataLength - currentDataOffset);
}
byte[] data = new byte[length];
- CarbonUnsafe.unsafe.copyMemory(dataPageMemoryBlock.getBaseObject(),
+ CarbonUnsafe.getUnsafe().copyMemory(dataPageMemoryBlock.getBaseObject(),
dataPageMemoryBlock.getBaseOffset() + currentDataOffset, data,
CarbonUnsafe.BYTE_ARRAY_OFFSET, length);
return data;
@@ -200,13 +200,13 @@ public class UnsafeVariableLengthDimesionDataChunkStore
// Subtract the current row offset + 2 bytes(to skip the data length) with next row offset
// else subtract the current row offset
// with complete data length get the offset of set of data
- int currentDataOffset = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
- dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets + (index
- * CarbonCommonConstants.INT_SIZE_IN_BYTE));
+ int currentDataOffset = CarbonUnsafe.getUnsafe().getInt(dataPageMemoryBlock.getBaseObject(),
+ dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets + ((long)index
+ * CarbonCommonConstants.INT_SIZE_IN_BYTE * 1L));
short length = 0;
// calculating the length of data
if (index < numberOfRows - 1) {
- int OffsetOfNextdata = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
+ int OffsetOfNextdata = CarbonUnsafe.getUnsafe().getInt(dataPageMemoryBlock.getBaseObject(),
dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets + ((index + 1)
* CarbonCommonConstants.INT_SIZE_IN_BYTE));
length = (short) (OffsetOfNextdata - (currentDataOffset
@@ -220,7 +220,7 @@ public class UnsafeVariableLengthDimesionDataChunkStore
int compareResult;
int compareLength = Math.min(length , compareValue.length);
for (int i = 0; i < compareLength; i++) {
- compareResult = (CarbonUnsafe.unsafe.getByte(dataPageMemoryBlock.getBaseObject(),
+ compareResult = (CarbonUnsafe.getUnsafe().getByte(dataPageMemoryBlock.getBaseObject(),
dataPageMemoryBlock.getBaseOffset() + currentDataOffset) & 0xff) - (compareValue[i]
& 0xff);
// if compare result is not equal we can break
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForShort.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForShort.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForShort.java
index 7303b66..a91d6bc 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForShort.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForShort.java
@@ -117,7 +117,7 @@ public class BlockIndexerStorageForShort implements IndexStorage<short[]> {
} else {
list.add(rowIds[i - 1]);
}
- double compressionPercentage = (((list.size() + map.size()) * 100) / rowIds.length);
+ int compressionPercentage = (((list.size() + map.size()) * 100) / rowIds.length);
if (compressionPercentage > 70) {
rowIdPage = rowIds;
} else {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
index 3477834..3191200 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
@@ -367,6 +367,9 @@ public final class FileFactory {
return path.delete();
}
File[] files = path.listFiles();
+ if (null == files) {
+ return true;
+ }
for (int i = 0; i < files.length; i++) {
deleteAllFilesOfDir(files[i]);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileHolderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileHolderImpl.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileHolderImpl.java
index 8e6aafc..c867f75 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileHolderImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileHolderImpl.java
@@ -217,7 +217,13 @@ public class FileHolderImpl implements FileHolder {
@Override public DataInputStream getDataInputStream(String filePath, long offset)
throws IOException {
FileInputStream stream = new FileInputStream(filePath);
- stream.skip(offset);
+ long skipped = stream.skip(offset);
+ long toSkip = offset - skipped;
+ while (toSkip <= 0) {
+ skipped = stream.skip(toSkip);
+ toSkip = toSkip - skipped;
+ }
+
return new DataInputStream(new BufferedInputStream(stream));
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
index a0eeb44..8b44f07 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
@@ -410,6 +410,8 @@ public abstract class ColumnPage {
case DECIMAL:
putDecimal(rowId, BigDecimal.ZERO);
break;
+ default:
+ throw new IllegalArgumentException("unsupported data type: " + dataType);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java
index f019ae8..06d952d 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java
@@ -82,40 +82,40 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
@Override
public void putByte(int rowId, byte value) {
long offset = rowId << byteBits;
- CarbonUnsafe.unsafe.putByte(baseAddress, baseOffset + offset, value);
+ CarbonUnsafe.getUnsafe().putByte(baseAddress, baseOffset + offset, value);
}
@Override
public void putShort(int rowId, short value) {
long offset = rowId << shortBits;
- CarbonUnsafe.unsafe.putShort(baseAddress, baseOffset + offset, value);
+ CarbonUnsafe.getUnsafe().putShort(baseAddress, baseOffset + offset, value);
}
@Override
public void putShortInt(int rowId, int value) {
byte[] data = ByteUtil.to3Bytes(value);
- long offset = rowId * 3;
- CarbonUnsafe.unsafe.putByte(baseAddress, baseOffset + offset, data[0]);
- CarbonUnsafe.unsafe.putByte(baseAddress, baseOffset + offset + 1, data[1]);
- CarbonUnsafe.unsafe.putByte(baseAddress, baseOffset + offset + 2, data[2]);
+ long offset = rowId * 3L;
+ CarbonUnsafe.getUnsafe().putByte(baseAddress, baseOffset + offset, data[0]);
+ CarbonUnsafe.getUnsafe().putByte(baseAddress, baseOffset + offset + 1, data[1]);
+ CarbonUnsafe.getUnsafe().putByte(baseAddress, baseOffset + offset + 2, data[2]);
}
@Override
public void putInt(int rowId, int value) {
long offset = rowId << intBits;
- CarbonUnsafe.unsafe.putInt(baseAddress, baseOffset + offset, value);
+ CarbonUnsafe.getUnsafe().putInt(baseAddress, baseOffset + offset, value);
}
@Override
public void putLong(int rowId, long value) {
long offset = rowId << longBits;
- CarbonUnsafe.unsafe.putLong(baseAddress, baseOffset + offset, value);
+ CarbonUnsafe.getUnsafe().putLong(baseAddress, baseOffset + offset, value);
}
@Override
public void putDouble(int rowId, double value) {
long offset = rowId << doubleBits;
- CarbonUnsafe.unsafe.putDouble(baseAddress, baseOffset + offset, value);
+ CarbonUnsafe.getUnsafe().putDouble(baseAddress, baseOffset + offset, value);
}
@Override
@@ -135,47 +135,47 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
@Override
public byte getByte(int rowId) {
long offset = rowId << byteBits;
- return CarbonUnsafe.unsafe.getByte(baseAddress, baseOffset + offset);
+ return CarbonUnsafe.getUnsafe().getByte(baseAddress, baseOffset + offset);
}
@Override
public short getShort(int rowId) {
long offset = rowId << shortBits;
- return CarbonUnsafe.unsafe.getShort(baseAddress, baseOffset + offset);
+ return CarbonUnsafe.getUnsafe().getShort(baseAddress, baseOffset + offset);
}
@Override
public int getShortInt(int rowId) {
- long offset = rowId * 3;
+ long offset = rowId * 3L;
byte[] data = new byte[3];
- data[0] = CarbonUnsafe.unsafe.getByte(baseAddress, baseOffset + offset);
- data[1] = CarbonUnsafe.unsafe.getByte(baseAddress, baseOffset + offset + 1);
- data[2] = CarbonUnsafe.unsafe.getByte(baseAddress, baseOffset + offset + 2);
+ data[0] = CarbonUnsafe.getUnsafe().getByte(baseAddress, baseOffset + offset);
+ data[1] = CarbonUnsafe.getUnsafe().getByte(baseAddress, baseOffset + offset + 1);
+ data[2] = CarbonUnsafe.getUnsafe().getByte(baseAddress, baseOffset + offset + 2);
return ByteUtil.valueOf3Bytes(data, 0);
}
@Override
public int getInt(int rowId) {
long offset = rowId << intBits;
- return CarbonUnsafe.unsafe.getInt(baseAddress, baseOffset + offset);
+ return CarbonUnsafe.getUnsafe().getInt(baseAddress, baseOffset + offset);
}
@Override
public long getLong(int rowId) {
long offset = rowId << longBits;
- return CarbonUnsafe.unsafe.getLong(baseAddress, baseOffset + offset);
+ return CarbonUnsafe.getUnsafe().getLong(baseAddress, baseOffset + offset);
}
@Override
public float getFloat(int rowId) {
long offset = rowId << floatBits;
- return CarbonUnsafe.unsafe.getFloat(baseAddress, baseOffset + offset);
+ return CarbonUnsafe.getUnsafe().getFloat(baseAddress, baseOffset + offset);
}
@Override
public double getDouble(int rowId) {
long offset = rowId << doubleBits;
- return CarbonUnsafe.unsafe.getDouble(baseAddress, baseOffset + offset);
+ return CarbonUnsafe.getUnsafe().getDouble(baseAddress, baseOffset + offset);
}
@Override
@@ -192,7 +192,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
byte[] data = new byte[getPageSize()];
for (int i = 0; i < data.length; i++) {
long offset = i << byteBits;
- data[i] = CarbonUnsafe.unsafe.getByte(baseAddress, baseOffset + offset);
+ data[i] = CarbonUnsafe.getUnsafe().getByte(baseAddress, baseOffset + offset);
}
return data;
}
@@ -202,7 +202,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
short[] data = new short[getPageSize()];
for (int i = 0; i < data.length; i++) {
long offset = i << shortBits;
- data[i] = CarbonUnsafe.unsafe.getShort(baseAddress, baseOffset + offset);
+ data[i] = CarbonUnsafe.getUnsafe().getShort(baseAddress, baseOffset + offset);
}
return data;
}
@@ -210,7 +210,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
@Override
public byte[] getShortIntPage() {
byte[] data = new byte[pageSize * 3];
- CarbonUnsafe.unsafe.copyMemory(baseAddress, baseOffset,
+ CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset,
data, CarbonUnsafe.BYTE_ARRAY_OFFSET, data.length);
return data;
}
@@ -220,7 +220,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
int[] data = new int[getPageSize()];
for (int i = 0; i < data.length; i++) {
long offset = i << intBits;
- data[i] = CarbonUnsafe.unsafe.getInt(baseAddress, baseOffset + offset);
+ data[i] = CarbonUnsafe.getUnsafe().getInt(baseAddress, baseOffset + offset);
}
return data;
}
@@ -230,7 +230,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
long[] data = new long[getPageSize()];
for (int i = 0; i < data.length; i++) {
long offset = i << longBits;
- data[i] = CarbonUnsafe.unsafe.getLong(baseAddress, baseOffset + offset);
+ data[i] = CarbonUnsafe.getUnsafe().getLong(baseAddress, baseOffset + offset);
}
return data;
}
@@ -240,7 +240,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
float[] data = new float[getPageSize()];
for (int i = 0; i < data.length; i++) {
long offset = i << floatBits;
- data[i] = CarbonUnsafe.unsafe.getFloat(baseAddress, baseOffset + offset);
+ data[i] = CarbonUnsafe.getUnsafe().getFloat(baseAddress, baseOffset + offset);
}
return data;
}
@@ -250,7 +250,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
double[] data = new double[getPageSize()];
for (int i = 0; i < data.length; i++) {
long offset = i << doubleBits;
- data[i] = CarbonUnsafe.unsafe.getDouble(baseAddress, baseOffset + offset);
+ data[i] = CarbonUnsafe.getUnsafe().getDouble(baseAddress, baseOffset + offset);
}
return data;
}
@@ -267,43 +267,43 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
@Override
public void setBytePage(byte[] byteData) {
- CarbonUnsafe.unsafe.copyMemory(byteData, CarbonUnsafe.BYTE_ARRAY_OFFSET,
+ CarbonUnsafe.getUnsafe().copyMemory(byteData, CarbonUnsafe.BYTE_ARRAY_OFFSET,
baseAddress, baseOffset, byteData.length << byteBits);
}
@Override
public void setShortPage(short[] shortData) {
- CarbonUnsafe.unsafe.copyMemory(shortData, CarbonUnsafe.SHORT_ARRAY_OFFSET,
+ CarbonUnsafe.getUnsafe().copyMemory(shortData, CarbonUnsafe.SHORT_ARRAY_OFFSET,
baseAddress, baseOffset, shortData.length << shortBits);
}
@Override
public void setShortIntPage(byte[] shortIntData) {
- CarbonUnsafe.unsafe.copyMemory(shortIntData, CarbonUnsafe.BYTE_ARRAY_OFFSET,
+ CarbonUnsafe.getUnsafe().copyMemory(shortIntData, CarbonUnsafe.BYTE_ARRAY_OFFSET,
baseAddress, baseOffset, shortIntData.length);
}
@Override
public void setIntPage(int[] intData) {
- CarbonUnsafe.unsafe.copyMemory(intData, CarbonUnsafe.INT_ARRAY_OFFSET,
+ CarbonUnsafe.getUnsafe().copyMemory(intData, CarbonUnsafe.INT_ARRAY_OFFSET,
baseAddress, baseOffset, intData.length << intBits);
}
@Override
public void setLongPage(long[] longData) {
- CarbonUnsafe.unsafe.copyMemory(longData, CarbonUnsafe.LONG_ARRAY_OFFSET,
+ CarbonUnsafe.getUnsafe().copyMemory(longData, CarbonUnsafe.LONG_ARRAY_OFFSET,
baseAddress, baseOffset, longData.length << longBits);
}
@Override
public void setFloatPage(float[] floatData) {
- CarbonUnsafe.unsafe.copyMemory(floatData, CarbonUnsafe.FLOAT_ARRAY_OFFSET,
+ CarbonUnsafe.getUnsafe().copyMemory(floatData, CarbonUnsafe.FLOAT_ARRAY_OFFSET,
baseAddress, baseOffset, floatData.length << floatBits);
}
@Override
public void setDoublePage(double[] doubleData) {
- CarbonUnsafe.unsafe.copyMemory(doubleData, CarbonUnsafe.DOUBLE_ARRAY_OFFSET,
+ CarbonUnsafe.getUnsafe().copyMemory(doubleData, CarbonUnsafe.DOUBLE_ARRAY_OFFSET,
baseAddress, baseOffset, doubleData.length << doubleBits);
}
@@ -328,37 +328,37 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
case BYTE:
for (int i = 0; i < pageSize; i++) {
long offset = i << byteBits;
- codec.encode(i, CarbonUnsafe.unsafe.getByte(baseAddress, baseOffset + offset));
+ codec.encode(i, CarbonUnsafe.getUnsafe().getByte(baseAddress, baseOffset + offset));
}
break;
case SHORT:
for (int i = 0; i < pageSize; i++) {
long offset = i << shortBits;
- codec.encode(i, CarbonUnsafe.unsafe.getShort(baseAddress, baseOffset + offset));
+ codec.encode(i, CarbonUnsafe.getUnsafe().getShort(baseAddress, baseOffset + offset));
}
break;
case INT:
for (int i = 0; i < pageSize; i++) {
long offset = i << intBits;
- codec.encode(i, CarbonUnsafe.unsafe.getInt(baseAddress, baseOffset + offset));
+ codec.encode(i, CarbonUnsafe.getUnsafe().getInt(baseAddress, baseOffset + offset));
}
break;
case LONG:
for (int i = 0; i < pageSize; i++) {
long offset = i << longBits;
- codec.encode(i, CarbonUnsafe.unsafe.getLong(baseAddress, baseOffset + offset));
+ codec.encode(i, CarbonUnsafe.getUnsafe().getLong(baseAddress, baseOffset + offset));
}
break;
case FLOAT:
for (int i = 0; i < pageSize; i++) {
long offset = i << floatBits;
- codec.encode(i, CarbonUnsafe.unsafe.getFloat(baseAddress, baseOffset + offset));
+ codec.encode(i, CarbonUnsafe.getUnsafe().getFloat(baseAddress, baseOffset + offset));
}
break;
case DOUBLE:
for (int i = 0; i < pageSize; i++) {
long offset = i << doubleBits;
- codec.encode(i, CarbonUnsafe.unsafe.getDouble(baseAddress, baseOffset + offset));
+ codec.encode(i, CarbonUnsafe.getUnsafe().getDouble(baseAddress, baseOffset + offset));
}
break;
default:
@@ -366,8 +366,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
}
}
- @Override
- public byte[] compress(Compressor compressor) throws MemoryException, IOException {
+ @Override public byte[] compress(Compressor compressor) throws MemoryException, IOException {
if (UnsafeMemoryManager.isOffHeap()) {
// use raw compression and copy to byte[]
int inputSize = pageSize * dataType.getSizeInBytes();
@@ -377,13 +376,13 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
long outSize = compressor.rawCompress(baseOffset, inputSize, compressed.getBaseOffset());
assert outSize < Integer.MAX_VALUE;
byte[] output = new byte[(int) outSize];
- CarbonUnsafe.unsafe.copyMemory(compressed.getBaseObject(), compressed.getBaseOffset(), output,
- CarbonUnsafe.BYTE_ARRAY_OFFSET, outSize);
+ CarbonUnsafe.getUnsafe()
+ .copyMemory(compressed.getBaseObject(), compressed.getBaseOffset(), output,
+ CarbonUnsafe.BYTE_ARRAY_OFFSET, outSize);
UnsafeMemoryManager.INSTANCE.freeMemory(taskId, compressed);
return output;
} else {
return super.compress(compressor);
}
}
-
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java
index c89d29d..c7d21ef 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java
@@ -95,7 +95,7 @@ public class UnsafeVarLengthColumnPage extends VarLengthColumnPageBase {
if (totalLength + requestSize > capacity) {
int newSize = 2 * capacity;
MemoryBlock newBlock = UnsafeMemoryManager.allocateMemoryWithRetry(taskId, newSize);
- CarbonUnsafe.unsafe.copyMemory(baseAddress, baseOffset,
+ CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset,
newBlock.getBaseObject(), newBlock.getBaseOffset(), capacity);
UnsafeMemoryManager.INSTANCE.freeMemory(taskId, memoryBlock);
memoryBlock = newBlock;
@@ -117,7 +117,7 @@ public class UnsafeVarLengthColumnPage extends VarLengthColumnPageBase {
} catch (MemoryException e) {
throw new RuntimeException(e);
}
- CarbonUnsafe.unsafe.copyMemory(bytes, CarbonUnsafe.BYTE_ARRAY_OFFSET + offset,
+ CarbonUnsafe.getUnsafe().copyMemory(bytes, CarbonUnsafe.BYTE_ARRAY_OFFSET + offset,
baseAddress, baseOffset + rowOffset[rowId], length);
}
@@ -129,7 +129,7 @@ public class UnsafeVarLengthColumnPage extends VarLengthColumnPageBase {
public BigDecimal getDecimal(int rowId) {
int length = rowOffset[rowId + 1] - rowOffset[rowId];
byte[] bytes = new byte[length];
- CarbonUnsafe.unsafe.copyMemory(baseAddress, baseOffset + rowOffset[rowId],
+ CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset[rowId],
bytes, CarbonUnsafe.BYTE_ARRAY_OFFSET, length);
return decimalConverter.getDecimal(bytes);
@@ -141,7 +141,7 @@ public class UnsafeVarLengthColumnPage extends VarLengthColumnPageBase {
for (int rowId = 0; rowId < pageSize; rowId++) {
int length = rowOffset[rowId + 1] - rowOffset[rowId];
byte[] rowData = new byte[length];
- CarbonUnsafe.unsafe.copyMemory(baseAddress, baseOffset + rowOffset[rowId],
+ CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset[rowId],
rowData, CarbonUnsafe.BYTE_ARRAY_OFFSET, length);
bytes[rowId] = rowData;
}
@@ -150,7 +150,7 @@ public class UnsafeVarLengthColumnPage extends VarLengthColumnPageBase {
@Override
void copyBytes(int rowId, byte[] dest, int destOffset, int length) {
- CarbonUnsafe.unsafe.copyMemory(baseAddress, baseOffset + rowOffset[rowId],
+ CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset[rowId],
dest, CarbonUnsafe.BYTE_ARRAY_OFFSET + destOffset, length);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DeltaIntegralCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DeltaIntegralCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DeltaIntegralCodec.java
index ba61cd3..80545b9 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DeltaIntegralCodec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DeltaIntegralCodec.java
@@ -44,8 +44,8 @@ public class DeltaIntegralCodec extends AdaptiveCompressionCodec {
return new DeltaIntegralCodec(srcDataType, targetDataType, stats, compressor);
}
- private DeltaIntegralCodec(DataType srcDataType, DataType targetDataType,
- SimpleStatsResult stats, Compressor compressor) {
+ private DeltaIntegralCodec(DataType srcDataType, DataType targetDataType, SimpleStatsResult stats,
+ Compressor compressor) {
super(srcDataType, targetDataType, stats, compressor);
switch (srcDataType) {
case BYTE:
@@ -62,8 +62,11 @@ public class DeltaIntegralCodec extends AdaptiveCompressionCodec {
break;
case FLOAT:
case DOUBLE:
- max = (long)((double) stats.getMax());
+ max = (long) ((double) stats.getMax());
break;
+ default:
+ throw new UnsupportedOperationException(
+ "unsupported data type for Delta compress: " + srcDataType);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/RLECodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/RLECodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/RLECodec.java
index 46c5c58..dda89e0 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/RLECodec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/RLECodec.java
@@ -326,6 +326,8 @@ public class RLECodec implements ColumnPageCodec {
case LONG:
decodeLongPage(in, resultPage);
break;
+ default:
+ throw new RuntimeException("unsupported datatype:" + dataType);
}
return resultPage;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java
index 918039a..927ab5f 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java
@@ -88,6 +88,9 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector, Si
instance.scale = meta.getScale();
instance.precision = meta.getPrecision();
break;
+ default:
+ throw new UnsupportedOperationException(
+ "unsupported data type for stats collection: " + meta.getSrcDataType());
}
return instance;
}
@@ -125,6 +128,9 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector, Si
instance.scale = -1;
instance.precision = -1;
break;
+ default:
+ throw new UnsupportedOperationException(
+ "unsupported data type for Stats collection: " + meta.getType());
}
return instance;
}
@@ -159,6 +165,10 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector, Si
decimal = scale;
this.scale = scale;
this.precision = precision;
+ break;
+ default:
+ throw new UnsupportedOperationException(
+ "unsupported data type for Stats collection: " + dataType);
}
}
@@ -192,6 +202,9 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector, Si
minDecimal = (minDecimal.compareTo(zeroDecimal) < 0) ? minDecimal : zeroDecimal;
}
break;
+ default:
+ throw new UnsupportedOperationException(
+ "unsupported data type for Stats collection: " + dataType);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/row/LoadStatusType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/row/LoadStatusType.java b/core/src/main/java/org/apache/carbondata/core/datastore/row/LoadStatusType.java
index cbdeadb..d3c5a94 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/row/LoadStatusType.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/row/LoadStatusType.java
@@ -34,8 +34,4 @@ public enum LoadStatusType {
public String getMessage() {
return message;
}
-
- public void setMessage(String message) {
- this.message = message;
- }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClientHandler.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClientHandler.java b/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClientHandler.java
index 9922523..01ef59a 100644
--- a/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClientHandler.java
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClientHandler.java
@@ -59,7 +59,7 @@ public class DictionaryClientHandler extends ChannelInboundHandlerAdapter {
DictionaryMessage key = new DictionaryMessage();
key.readData(data);
data.release();
- responseMsgQueue.offer(key);
+ responseMsgQueue.add(key);
} catch (Exception e) {
LOGGER.error(e);
throw e;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGenerator.java b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGenerator.java
index 9246830..221f9b7 100644
--- a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGenerator.java
@@ -16,6 +16,7 @@
*/
package org.apache.carbondata.core.dictionary.generator;
+import java.util.Iterator;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@@ -71,8 +72,14 @@ public class ServerDictionaryGenerator implements DictionaryGenerator<Integer, D
}
public void writeDictionaryData() throws Exception {
- for (String tableUniqueName: tableMap.keySet()) {
- TableDictionaryGenerator generator = tableMap.get(tableUniqueName);
+ final Iterator<Map.Entry<String, TableDictionaryGenerator>> iterator =
+ tableMap.entrySet().iterator();
+ String tableUniqueName;
+ TableDictionaryGenerator generator;
+ while (iterator.hasNext()) {
+ Map.Entry<String, TableDictionaryGenerator> entry = iterator.next();
+ tableUniqueName = entry.getKey();
+ generator = entry.getValue();
generator.writeDictionaryData(tableUniqueName);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGenerator.java b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGenerator.java
index 56ed7b9..ee3a2fe 100644
--- a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGenerator.java
@@ -16,14 +16,11 @@
*/
package org.apache.carbondata.core.dictionary.generator;
-import java.util.ArrayList;
-import java.util.List;
+import java.io.IOException;
import java.util.Map;
-import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import org.apache.carbondata.common.logging.LogService;
@@ -83,7 +80,6 @@ public class TableDictionaryGenerator
@Override public void writeDictionaryData(String tableUniqueName) {
int numOfCores = 1;
- final String tableName = tableUniqueName;
try {
numOfCores = Integer.parseInt(CarbonProperties.getInstance()
.getProperty(CarbonCommonConstants.NUM_CORES_LOADING,
@@ -92,16 +88,9 @@ public class TableDictionaryGenerator
numOfCores = Integer.parseInt(CarbonCommonConstants.NUM_CORES_DEFAULT_VAL);
}
long start = System.currentTimeMillis();
- List<Future<Void>> taskSubmitList =
- new ArrayList<>(columnMap.size());
ExecutorService executorService = Executors.newFixedThreadPool(numOfCores);
- for (final DictionaryGenerator generator: columnMap.values()) {
- taskSubmitList.add(executorService.submit(new Callable<Void>() {
- @Override public Void call() throws Exception {
- ((DictionaryWriter) (generator)).writeDictionaryData(tableName);
- return null;
- }
- }));
+ for (final DictionaryGenerator generator : columnMap.values()) {
+ executorService.execute(new WriteDictionaryDataRunnable(generator, tableUniqueName));
}
try {
@@ -125,4 +114,22 @@ public class TableDictionaryGenerator
}
}
}
+
+ private static class WriteDictionaryDataRunnable implements Runnable {
+ private final DictionaryGenerator generator;
+ private final String tableUniqueName;
+
+ public WriteDictionaryDataRunnable(DictionaryGenerator generator, String tableUniqueName) {
+ this.generator = generator;
+ this.tableUniqueName = tableUniqueName;
+ }
+
+ @Override public void run() {
+ try {
+ ((DictionaryWriter)generator).writeDictionaryData(tableUniqueName);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/dictionary/server/DictionaryServer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/server/DictionaryServer.java b/core/src/main/java/org/apache/carbondata/core/dictionary/server/DictionaryServer.java
index 84f2a0d..617038e 100644
--- a/core/src/main/java/org/apache/carbondata/core/dictionary/server/DictionaryServer.java
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/server/DictionaryServer.java
@@ -47,7 +47,7 @@ public class DictionaryServer {
private EventLoopGroup worker;
private int port;
private static Object lock = new Object();
- private static DictionaryServer INSTANCE = null;
+ private static volatile DictionaryServer INSTANCE = null;
private DictionaryServer(int port) {
startServer(port);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java b/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java
index 66da4d0..c3a72f0 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java
@@ -31,7 +31,7 @@ import org.apache.hadoop.fs.RemoteIterator;
*/
public class Blocklet implements Serializable {
- private Path path;
+ private String path;
private String segmentId;
@@ -44,12 +44,12 @@ public class Blocklet implements Serializable {
private String[] location;
public Blocklet(String path, String blockletId) {
- this.path = new Path(path);
+ this.path = path;
this.blockletId = blockletId;
}
public Path getPath() {
- return path;
+ return new Path(path);
}
public String getBlockletId() {
@@ -65,8 +65,9 @@ public class Blocklet implements Serializable {
}
public void updateLocations() throws IOException {
- FileSystem fs = path.getFileSystem(FileFactory.getConfiguration());
- RemoteIterator<LocatedFileStatus> iter = fs.listLocatedStatus(path);
+ Path fspath = new Path(path);
+ FileSystem fs = fspath.getFileSystem(FileFactory.getConfiguration());
+ RemoteIterator<LocatedFileStatus> iter = fs.listLocatedStatus(fspath);
LocatedFileStatus fileStatus = iter.next();
location = fileStatus.getBlockLocations()[0].getHosts();
length = fileStatus.getLen();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/indexstore/UnsafeMemoryDMStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/UnsafeMemoryDMStore.java b/core/src/main/java/org/apache/carbondata/core/indexstore/UnsafeMemoryDMStore.java
index eb67294..5d17426 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/UnsafeMemoryDMStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/UnsafeMemoryDMStore.java
@@ -25,7 +25,7 @@ import org.apache.carbondata.core.memory.UnsafeMemoryManager;
import org.apache.carbondata.core.util.ThreadLocalTaskInfo;
import static org.apache.carbondata.core.memory.CarbonUnsafe.BYTE_ARRAY_OFFSET;
-import static org.apache.carbondata.core.memory.CarbonUnsafe.unsafe;
+import static org.apache.carbondata.core.memory.CarbonUnsafe.getUnsafe;
/**
* Store the data map row @{@link DataMapRow} data to unsafe.
@@ -67,7 +67,7 @@ public class UnsafeMemoryDMStore {
if (runningLength + rowSize >= allocatedSize) {
MemoryBlock allocate =
UnsafeMemoryManager.allocateMemoryWithRetry(taskId, allocatedSize + capacity);
- unsafe.copyMemory(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset(),
+ getUnsafe().copyMemory(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset(),
allocate.getBaseObject(), allocate.getBaseOffset(), runningLength);
UnsafeMemoryManager.INSTANCE.freeMemory(taskId, memoryBlock);
allocatedSize = allocatedSize + capacity;
@@ -104,51 +104,57 @@ public class UnsafeMemoryDMStore {
case FIXED:
switch (schema.getDataType()) {
case BYTE:
- unsafe.putByte(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength,
- row.getByte(index));
+ getUnsafe()
+ .putByte(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength,
+ row.getByte(index));
runningLength += row.getSizeInBytes(index);
break;
case SHORT:
- unsafe
+ getUnsafe()
.putShort(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength,
row.getShort(index));
runningLength += row.getSizeInBytes(index);
break;
case INT:
- unsafe.putInt(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength,
- row.getInt(index));
+ getUnsafe()
+ .putInt(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength,
+ row.getInt(index));
runningLength += row.getSizeInBytes(index);
break;
case LONG:
- unsafe.putLong(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength,
- row.getLong(index));
+ getUnsafe()
+ .putLong(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength,
+ row.getLong(index));
runningLength += row.getSizeInBytes(index);
break;
case FLOAT:
- unsafe
+ getUnsafe()
.putFloat(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength,
row.getFloat(index));
runningLength += row.getSizeInBytes(index);
break;
case DOUBLE:
- unsafe
+ getUnsafe()
.putDouble(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength,
row.getDouble(index));
runningLength += row.getSizeInBytes(index);
break;
case BYTE_ARRAY:
byte[] data = row.getByteArray(index);
- unsafe.copyMemory(data, BYTE_ARRAY_OFFSET, memoryBlock.getBaseObject(),
+ getUnsafe().copyMemory(data, BYTE_ARRAY_OFFSET, memoryBlock.getBaseObject(),
memoryBlock.getBaseOffset() + runningLength, data.length);
runningLength += row.getSizeInBytes(index);
break;
+ default:
+ throw new UnsupportedOperationException(
+ "unsupported data type for unsafe storage: " + schema.getDataType());
}
break;
case VARIABLE:
byte[] data = row.getByteArray(index);
- unsafe.putShort(memoryBlock.getBaseOffset() + runningLength, (short) data.length);
+ getUnsafe().putShort(memoryBlock.getBaseOffset() + runningLength, (short) data.length);
runningLength += 2;
- unsafe.copyMemory(data, BYTE_ARRAY_OFFSET, memoryBlock.getBaseObject(),
+ getUnsafe().copyMemory(data, BYTE_ARRAY_OFFSET, memoryBlock.getBaseObject(),
memoryBlock.getBaseOffset() + runningLength, data.length);
runningLength += data.length;
break;
@@ -160,6 +166,9 @@ public class UnsafeMemoryDMStore {
addToUnsafe(childSchemas[i], struct, i);
}
break;
+ default:
+ throw new UnsupportedOperationException(
+ "unsupported data type for unsafe storage: " + schema.getDataType());
}
}
@@ -172,7 +181,7 @@ public class UnsafeMemoryDMStore {
if (runningLength < allocatedSize) {
MemoryBlock allocate =
UnsafeMemoryManager.allocateMemoryWithRetry(taskId, runningLength);
- unsafe.copyMemory(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset(),
+ getUnsafe().copyMemory(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset(),
allocate.getBaseObject(), allocate.getBaseOffset(), runningLength);
UnsafeMemoryManager.INSTANCE.freeMemory(taskId, memoryBlock);
memoryBlock = allocate;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
index 2e82c46..928c78b 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
@@ -24,6 +24,7 @@ import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.BitSet;
import java.util.Comparator;
import java.util.List;
@@ -31,6 +32,7 @@ import java.util.List;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.cache.Cacheable;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datamap.DataMapDistributable;
import org.apache.carbondata.core.datamap.dev.DataMap;
import org.apache.carbondata.core.datastore.IndexKey;
@@ -131,7 +133,8 @@ public class BlockletDataMap implements DataMap, Cacheable {
row.setInt(blockletInfo.getNumberOfRows(), ordinal++);
// add file path
- byte[] filePathBytes = filePath.getBytes();
+ byte[] filePathBytes =
+ filePath.getBytes(CarbonCommonConstants.DEFAULT_CHARSET_CLASS);
row.setByteArray(filePathBytes, ordinal++);
// add pages
@@ -244,10 +247,11 @@ public class BlockletDataMap implements DataMap, Cacheable {
}
if (LOGGER.isDebugEnabled()) {
LOGGER.debug(
- "Successfully retrieved the start and end key" + "Dictionary Start Key: " + searchStartKey
- .getDictionaryKeys() + "No Dictionary Start Key " + searchStartKey
- .getNoDictionaryKeys() + "Dictionary End Key: " + searchEndKey.getDictionaryKeys()
- + "No Dictionary End Key " + searchEndKey.getNoDictionaryKeys());
+ "Successfully retrieved the start and end key" + "Dictionary Start Key: " + Arrays
+ .toString(searchStartKey.getDictionaryKeys()) + "No Dictionary Start Key " + Arrays
+ .toString(searchStartKey.getNoDictionaryKeys()) + "Dictionary End Key: " + Arrays
+ .toString(searchEndKey.getDictionaryKeys()) + "No Dictionary End Key " + Arrays
+ .toString(searchEndKey.getNoDictionaryKeys()));
}
if (filterExp == null) {
int rowCount = unsafeMemoryDMStore.getRowCount();
@@ -284,8 +288,9 @@ public class BlockletDataMap implements DataMap, Cacheable {
}
private Blocklet createBlocklet(DataMapRow row, int blockletId) {
- Blocklet blocklet =
- new Blocklet(new String(row.getByteArray(FILE_PATH_INDEX)), blockletId + "");
+ Blocklet blocklet = new Blocklet(
+ new String(row.getByteArray(FILE_PATH_INDEX), CarbonCommonConstants.DEFAULT_CHARSET_CLASS),
+ blockletId + "");
BlockletDetailInfo detailInfo = new BlockletDetailInfo();
detailInfo.setRowCount(row.getInt(ROW_COUNT_INDEX));
detailInfo.setPagesCount(row.getShort(PAGE_COUNT_INDEX));
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java
index 5509c75..8896e5d 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java
@@ -52,7 +52,7 @@ public class BlockletDataRefNodeWrapper implements DataRefNode {
@Override public DataRefNode getNextDataRefNode() {
if (index + 1 < blockInfos.size()) {
- new BlockletDataRefNodeWrapper(blockInfos, index + 1, dimensionLens);
+ return new BlockletDataRefNodeWrapper(blockInfos, index + 1, dimensionLens);
}
return null;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/indexstore/row/UnsafeDataMapRow.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/row/UnsafeDataMapRow.java b/core/src/main/java/org/apache/carbondata/core/indexstore/row/UnsafeDataMapRow.java
index c398115..2c76990 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/row/UnsafeDataMapRow.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/row/UnsafeDataMapRow.java
@@ -21,7 +21,7 @@ import org.apache.carbondata.core.indexstore.schema.DataMapSchema;
import org.apache.carbondata.core.memory.MemoryBlock;
import static org.apache.carbondata.core.memory.CarbonUnsafe.BYTE_ARRAY_OFFSET;
-import static org.apache.carbondata.core.memory.CarbonUnsafe.unsafe;
+import static org.apache.carbondata.core.memory.CarbonUnsafe.getUnsafe;
/**
* Unsafe implementation of data map row.
@@ -43,14 +43,15 @@ public class UnsafeDataMapRow extends DataMapRow {
int position = getPosition(ordinal);
switch (schemas[ordinal].getSchemaType()) {
case VARIABLE:
- length = unsafe.getShort(block.getBaseObject(), block.getBaseOffset() + pointer + position);
+ length =
+ getUnsafe().getShort(block.getBaseObject(), block.getBaseOffset() + pointer + position);
position += 2;
break;
default:
length = schemas[ordinal].getLength();
}
byte[] data = new byte[length];
- unsafe.copyMemory(block.getBaseObject(), block.getBaseOffset() + pointer + position, data,
+ getUnsafe().copyMemory(block.getBaseObject(), block.getBaseOffset() + pointer + position, data,
BYTE_ARRAY_OFFSET, data.length);
return data;
}
@@ -60,7 +61,8 @@ public class UnsafeDataMapRow extends DataMapRow {
int position = getPosition(ordinal);
switch (schemas[ordinal].getSchemaType()) {
case VARIABLE:
- length = unsafe.getShort(block.getBaseObject(), block.getBaseOffset() + pointer + position);
+ length =
+ getUnsafe().getShort(block.getBaseObject(), block.getBaseOffset() + pointer + position);
break;
default:
length = schemas[ordinal].getLength();
@@ -72,7 +74,8 @@ public class UnsafeDataMapRow extends DataMapRow {
int length;
switch (schemas[ordinal].getSchemaType()) {
case VARIABLE:
- length = unsafe.getShort(block.getBaseObject(), block.getBaseOffset() + pointer + position);
+ length =
+ getUnsafe().getShort(block.getBaseObject(), block.getBaseOffset() + pointer + position);
break;
default:
length = schemas[ordinal].getLength();
@@ -91,7 +94,7 @@ public class UnsafeDataMapRow extends DataMapRow {
}
@Override public int getInt(int ordinal) {
- return unsafe
+ return getUnsafe()
.getInt(block.getBaseObject(), block.getBaseOffset() + pointer + getPosition(ordinal));
}
@@ -104,7 +107,7 @@ public class UnsafeDataMapRow extends DataMapRow {
}
@Override public byte getByte(int ordinal) {
- return unsafe
+ return getUnsafe()
.getByte(block.getBaseObject(), block.getBaseOffset() + pointer + getPosition(ordinal));
}
@@ -113,7 +116,7 @@ public class UnsafeDataMapRow extends DataMapRow {
}
@Override public short getShort(int ordinal) {
- return unsafe
+ return getUnsafe()
.getShort(block.getBaseObject(), block.getBaseOffset() + pointer + getPosition(ordinal));
}
@@ -122,7 +125,7 @@ public class UnsafeDataMapRow extends DataMapRow {
}
@Override public long getLong(int ordinal) {
- return unsafe
+ return getUnsafe()
.getLong(block.getBaseObject(), block.getBaseOffset() + pointer + getPosition(ordinal));
}
@@ -131,7 +134,7 @@ public class UnsafeDataMapRow extends DataMapRow {
}
@Override public float getFloat(int ordinal) {
- return unsafe
+ return getUnsafe()
.getFloat(block.getBaseObject(), block.getBaseOffset() + pointer + getPosition(ordinal));
}
@@ -140,7 +143,7 @@ public class UnsafeDataMapRow extends DataMapRow {
}
@Override public double getDouble(int ordinal) {
- return unsafe
+ return getUnsafe()
.getDouble(block.getBaseObject(), block.getBaseOffset() + pointer + getPosition(ordinal));
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/keygenerator/columnar/impl/MultiDimKeyVarLengthEquiSplitGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/columnar/impl/MultiDimKeyVarLengthEquiSplitGenerator.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/columnar/impl/MultiDimKeyVarLengthEquiSplitGenerator.java
index f628bca..72b250b 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/columnar/impl/MultiDimKeyVarLengthEquiSplitGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/columnar/impl/MultiDimKeyVarLengthEquiSplitGenerator.java
@@ -109,29 +109,6 @@ public class MultiDimKeyVarLengthEquiSplitGenerator extends MultiDimKeyVarLength
dimBlockArray[j] = convertToArray(dimBlockSet[j]);
}
- int[][] splitDimArrayLocalIndexes = new int[splitDimArray.length][];
- for (int j = 0; j < splitDimArrayLocalIndexes.length; j++) {
- splitDimArrayLocalIndexes[j] = splitDimArray[j].length > 0 ?
- new int[] { 0, splitDimArray[j][1] - splitDimArray[j][0] } :
- new int[0];
- }
-
- int[][][] byteRangesForDims = new int[byteRangesForKeys.length][][];
- for (int j = 0; j < byteRangesForKeys.length; j++) {
- if (dimBlockArray[j].length > 1) {
- int[] bArray1 = splitDimArrayLocalIndexes[dimBlockArray[j][0]];
- byteRangesForDims[j] = new int[2][2];
- byteRangesForDims[j][0] =
- new int[] { bArray1[bArray1.length - 1], bArray1[bArray1.length - 1] };
- byteRangesForDims[j][1] = new int[] { 0,
- (byteRangesForKeys[j][byteRangesForKeys[j].length - 1] - byteRangesForKeys[j][0]) - 1 };
- } else {
- byteRangesForDims[j] = new int[1][1];
- int[] bArray1 = splitDimArray[dimBlockArray[j][0]];
- byteRangesForDims[j][0] = new int[] { byteRangesForKeys[j][0] - bArray1[0],
- byteRangesForKeys[j][1] - bArray1[0] };
- }
- }
blockKeySize = new int[splitDimArray.length];
for (int j = 0; j < blockKeySize.length; j++) {
[2/3] carbondata git commit: [Review][CARBONDATA-1386] fixed findbugs
errors in carbondata-core
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/keygenerator/columnar/impl/MultiDimKeyVarLengthVariableSplitGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/columnar/impl/MultiDimKeyVarLengthVariableSplitGenerator.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/columnar/impl/MultiDimKeyVarLengthVariableSplitGenerator.java
index cb52045..1ed4577 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/columnar/impl/MultiDimKeyVarLengthVariableSplitGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/columnar/impl/MultiDimKeyVarLengthVariableSplitGenerator.java
@@ -110,29 +110,6 @@ public class MultiDimKeyVarLengthVariableSplitGenerator extends MultiDimKeyVarLe
dimBlockArray[j] = convertToArray(dimBlockSet[j]);
}
- int[][] splitDimArrayLocalIndexes = new int[splitDimArray.length][];
- for (int j = 0; j < splitDimArrayLocalIndexes.length; j++) {
- splitDimArrayLocalIndexes[j] = splitDimArray[j].length > 0 ?
- new int[] { 0, splitDimArray[j][1] - splitDimArray[j][0] } :
- new int[0];
- }
-
- int[][][] byteRangesForDims = new int[byteRangesForKeys.length][][];
- for (int j = 0; j < byteRangesForKeys.length; j++) {
- if (dimBlockArray[j].length > 1) {
- int[] bArray1 = splitDimArrayLocalIndexes[dimBlockArray[j][0]];
- byteRangesForDims[j] = new int[2][2];
- byteRangesForDims[j][0] =
- new int[] { bArray1[bArray1.length - 1], bArray1[bArray1.length - 1] };
- byteRangesForDims[j][1] = new int[] { 0,
- (byteRangesForKeys[j][byteRangesForKeys[j].length - 1] - byteRangesForKeys[j][0]) - 1 };
- } else {
- byteRangesForDims[j] = new int[1][1];
- int[] bArray1 = splitDimArray[dimBlockArray[j][0]];
- byteRangesForDims[j][0] = new int[] { byteRangesForKeys[j][0] - bArray1[0],
- byteRangesForKeys[j][1] - bArray1[0] };
- }
- }
blockKeySize = new int[splitDimArray.length];
for (int j = 0; j < blockKeySize.length; j++) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
index 9fc2c25..0d7cb6c 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
@@ -139,7 +139,7 @@ public class DateDirectDictionaryGenerator implements DirectDictionaryGenerator
private int generateDirectSurrogateKeyForNonTimestampType(String memberStr) {
long timeValue = -1;
try {
- timeValue = Long.valueOf(memberStr) / 1000;
+ timeValue = Long.parseLong(memberStr) / 1000;
} catch (NumberFormatException e) {
if (LOGGER.isDebugEnabled()) {
LOGGER.debug(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
index 5c7ba07..6a0b9e6 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
@@ -199,7 +199,7 @@ public class TimeStampDirectDictionaryGenerator implements DirectDictionaryGener
private int generateDirectSurrogateKeyForNonTimestampType(String memberStr) {
long timeValue = -1;
try {
- timeValue = Long.valueOf(memberStr) / 1000;
+ timeValue = Long.parseLong(memberStr) / 1000;
} catch (NumberFormatException e) {
if (LOGGER.isDebugEnabled()) {
LOGGER.debug(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java b/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java
index aa24e33..752c70d 100644
--- a/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java
+++ b/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java
@@ -45,7 +45,7 @@ public class HdfsFileLock extends AbstractCarbonLock {
private DataOutputStream dataOutputStream;
- public static String tmpPath;
+ private static String tmpPath;
static {
Configuration conf = new Configuration(true);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/locks/ZooKeeperLocking.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/locks/ZooKeeperLocking.java b/core/src/main/java/org/apache/carbondata/core/locks/ZooKeeperLocking.java
index 24bc5b0..5e83aa4 100644
--- a/core/src/main/java/org/apache/carbondata/core/locks/ZooKeeperLocking.java
+++ b/core/src/main/java/org/apache/carbondata/core/locks/ZooKeeperLocking.java
@@ -74,6 +74,14 @@ public class ZooKeeperLocking extends AbstractCarbonLock {
lockFile);
}
+ public static void initialize() {
+ String zooKeeperUrl =
+ CarbonProperties.getInstance().getProperty(CarbonCommonConstants.ZOOKEEPER_URL);
+ if (null == zk) {
+ zk = ZookeeperInit.getInstance(zooKeeperUrl).getZookeeper();
+ }
+ }
+
/**
* @param lockLocation
* @param lockFile
@@ -82,9 +90,7 @@ public class ZooKeeperLocking extends AbstractCarbonLock {
this.lockName = lockFile;
this.tableIdFolder = zooKeeperLocation + CarbonCommonConstants.FILE_SEPARATOR + lockLocation;
- String zooKeeperUrl =
- CarbonProperties.getInstance().getProperty(CarbonCommonConstants.ZOOKEEPER_URL);
- zk = ZookeeperInit.getInstance(zooKeeperUrl).getZookeeper();
+ initialize();
this.lockTypeFolder = zooKeeperLocation + CarbonCommonConstants.FILE_SEPARATOR + lockLocation
+ CarbonCommonConstants.FILE_SEPARATOR + lockFile;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/memory/CarbonUnsafe.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/memory/CarbonUnsafe.java b/core/src/main/java/org/apache/carbondata/core/memory/CarbonUnsafe.java
index 88e108c..74343f5 100644
--- a/core/src/main/java/org/apache/carbondata/core/memory/CarbonUnsafe.java
+++ b/core/src/main/java/org/apache/carbondata/core/memory/CarbonUnsafe.java
@@ -40,7 +40,7 @@ public final class CarbonUnsafe {
public static final boolean ISLITTLEENDIAN =
ByteOrder.nativeOrder().equals(ByteOrder.LITTLE_ENDIAN);
- public static Unsafe unsafe;
+ private static Unsafe unsafe;
static {
try {
@@ -66,4 +66,8 @@ public final class CarbonUnsafe {
DOUBLE_ARRAY_OFFSET = 0;
}
}
+
+ public static Unsafe getUnsafe() {
+ return unsafe;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/memory/IntPointerBuffer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/memory/IntPointerBuffer.java b/core/src/main/java/org/apache/carbondata/core/memory/IntPointerBuffer.java
index dadb1e4..5cd8e8d 100644
--- a/core/src/main/java/org/apache/carbondata/core/memory/IntPointerBuffer.java
+++ b/core/src/main/java/org/apache/carbondata/core/memory/IntPointerBuffer.java
@@ -67,7 +67,7 @@ public class IntPointerBuffer {
assert rowId >= 0 : "rowId (" + rowId + ") should >= 0";
assert rowId < length : "rowId (" + rowId + ") should < length (" + length + ")";
if (pointerBlock == null) {
- return CarbonUnsafe.unsafe.getInt(pointerMemoryBlock.getBaseObject(),
+ return CarbonUnsafe.getUnsafe().getInt(pointerMemoryBlock.getBaseObject(),
pointerMemoryBlock.getBaseOffset() + (rowId << 2));
}
return pointerBlock[rowId];
@@ -78,7 +78,7 @@ public class IntPointerBuffer {
pointerMemoryBlock =
UnsafeSortMemoryManager.allocateMemoryWithRetry(this.taskId, pointerBlock.length * 4);
for (int i = 0; i < pointerBlock.length; i++) {
- CarbonUnsafe.unsafe
+ CarbonUnsafe.getUnsafe()
.putInt(pointerMemoryBlock.getBaseObject(), pointerMemoryBlock.getBaseOffset() + i * 4,
pointerBlock[i]);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/memory/MemoryBlock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/memory/MemoryBlock.java b/core/src/main/java/org/apache/carbondata/core/memory/MemoryBlock.java
index ab9b3d4..74136c1 100644
--- a/core/src/main/java/org/apache/carbondata/core/memory/MemoryBlock.java
+++ b/core/src/main/java/org/apache/carbondata/core/memory/MemoryBlock.java
@@ -29,13 +29,6 @@ public class MemoryBlock extends MemoryLocation {
private final long length;
- /**
- * Optional page number; used when this MemoryBlock represents a page allocated by a
- * TaskMemoryManager. This field is public so that it can be modified by the TaskMemoryManager,
- * which lives in a different package.
- */
- public int pageNumber = -1;
-
public MemoryBlock(@Nullable Object obj, long offset, long length) {
super(obj, offset);
this.length = length;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryAllocator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryAllocator.java b/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryAllocator.java
index db38ff2..fcb0b88 100644
--- a/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryAllocator.java
+++ b/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryAllocator.java
@@ -25,9 +25,9 @@ public class UnsafeMemoryAllocator implements MemoryAllocator {
@Override
public MemoryBlock allocate(long size) throws OutOfMemoryError {
- long address = CarbonUnsafe.unsafe.allocateMemory(size);
+ long address = CarbonUnsafe.getUnsafe().allocateMemory(size);
// initializing memory with zero
- CarbonUnsafe.unsafe.setMemory(null, address, size, (byte) 0);
+ CarbonUnsafe.getUnsafe().setMemory(null, address, size, (byte) 0);
return new MemoryBlock(null, address, size);
}
@@ -35,6 +35,6 @@ public class UnsafeMemoryAllocator implements MemoryAllocator {
public void free(MemoryBlock memory) {
assert (memory.obj == null) :
"baseObject not null; are you trying to use the off-heap allocator to free on-heap memory?";
- CarbonUnsafe.unsafe.freeMemory(memory.offset);
+ CarbonUnsafe.getUnsafe().freeMemory(memory.offset);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java b/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java
index d433b5e..1190d56 100644
--- a/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java
@@ -114,11 +114,9 @@ public class UnsafeMemoryManager {
- memoryUsed));
}
- public void freeMemoryAll(long taskId) {
+ public synchronized void freeMemoryAll(long taskId) {
Set<MemoryBlock> memoryBlockSet = null;
- synchronized (INSTANCE) {
- memoryBlockSet = taskIdToMemoryBlockMap.remove(taskId);
- }
+ memoryBlockSet = taskIdToMemoryBlockMap.remove(taskId);
long occuppiedMemory = 0;
if (null != memoryBlockSet) {
Iterator<MemoryBlock> iterator = memoryBlockSet.iterator();
@@ -129,10 +127,8 @@ public class UnsafeMemoryManager {
allocator.free(memoryBlock);
}
}
- synchronized (INSTANCE) {
- memoryUsed -= occuppiedMemory;
- memoryUsed = memoryUsed < 0 ? 0 : memoryUsed;
- }
+ memoryUsed -= occuppiedMemory;
+ memoryUsed = memoryUsed < 0 ? 0 : memoryUsed;
if (LOGGER.isDebugEnabled()) {
LOGGER.debug(
"Freeing memory of size: " + occuppiedMemory + ": Current available memory is: " + (
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/memory/UnsafeSortMemoryManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/memory/UnsafeSortMemoryManager.java b/core/src/main/java/org/apache/carbondata/core/memory/UnsafeSortMemoryManager.java
index d975cd4..b4a200a 100644
--- a/core/src/main/java/org/apache/carbondata/core/memory/UnsafeSortMemoryManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/memory/UnsafeSortMemoryManager.java
@@ -152,11 +152,9 @@ public class UnsafeSortMemoryManager {
* when in case of task failure we need to clear all the memory occupied
* @param taskId
*/
- public void freeMemoryAll(long taskId) {
+ public synchronized void freeMemoryAll(long taskId) {
Set<MemoryBlock> memoryBlockSet = null;
- synchronized (INSTANCE) {
- memoryBlockSet = taskIdToMemoryBlockMap.remove(taskId);
- }
+ memoryBlockSet = taskIdToMemoryBlockMap.remove(taskId);
long occuppiedMemory = 0;
if (null != memoryBlockSet) {
Iterator<MemoryBlock> iterator = memoryBlockSet.iterator();
@@ -167,10 +165,8 @@ public class UnsafeSortMemoryManager {
allocator.free(memoryBlock);
}
}
- synchronized (INSTANCE) {
- memoryUsed -= occuppiedMemory;
- memoryUsed = memoryUsed < 0 ? 0 : memoryUsed;
- }
+ memoryUsed -= occuppiedMemory;
+ memoryUsed = memoryUsed < 0 ? 0 : memoryUsed;
if (LOGGER.isDebugEnabled()) {
LOGGER.debug(
"Freeing memory of size: " + occuppiedMemory + ": Current available memory is: " + (
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/metadata/ColumnPageCodecMeta.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/ColumnPageCodecMeta.java b/core/src/main/java/org/apache/carbondata/core/metadata/ColumnPageCodecMeta.java
index 591899e..54bc2ce 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/ColumnPageCodecMeta.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/ColumnPageCodecMeta.java
@@ -194,6 +194,8 @@ public class ColumnPageCodecMeta extends ValueEncoderMeta implements Serializabl
buffer.putInt(getScale());
buffer.putInt(getPrecision());
break;
+ default:
+ throw new RuntimeException("Unexpected type: " + srcDataType);
}
buffer.putInt(getDecimal());
buffer.put(getDataTypeSelected());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java
index 459eb24..0343e38 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java
@@ -69,11 +69,9 @@ public final class DecimalConverterFactory {
private ByteBuffer buffer = ByteBuffer.allocate(4);
- private int precision;
private int scale;
public DecimalIntConverter(int precision, int scale) {
- this.precision = precision;
this.scale = scale;
}
@@ -116,11 +114,9 @@ public final class DecimalConverterFactory {
private ByteBuffer buffer = ByteBuffer.allocate(8);
- private int precision;
private int scale;
public DecimalLongConverter(int precision, int scale) {
- this.precision = precision;
this.scale = scale;
}
@@ -147,7 +143,6 @@ public final class DecimalConverterFactory {
public class DecimalUnscaledConverter implements DecimalConverter {
- private int precision;
private int scale;
@@ -156,7 +151,6 @@ public final class DecimalConverterFactory {
private byte[] decimalBuffer = new byte[minBytesForPrecision[38]];
public DecimalUnscaledConverter(int precision, int scale) {
- this.precision = precision;
this.scale = scale;
this.numBytes = minBytesForPrecision[precision];
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java
index 2bd3198..0c807f6 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java
@@ -185,15 +185,15 @@ public class TableInfo implements Serializable, Writable {
return false;
}
TableInfo other = (TableInfo) obj;
- if ((databaseName == null && other.databaseName != null) || (databaseName != null
- && other.databaseName == null)) {
+ if (null == databaseName || null == other.databaseName) {
return false;
}
- if ((tableUniqueName == null && other.tableUniqueName != null) || (tableUniqueName != null
- && other.tableUniqueName == null)) {
+ if (null == tableUniqueName || null == other.tableUniqueName) {
return false;
- } else if (!tableUniqueName.equals(other.tableUniqueName)) {
+ }
+
+ if (!tableUniqueName.equals(other.tableUniqueName)) {
return false;
}
return true;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
index e92563b..57e1544 100644
--- a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
@@ -129,10 +129,6 @@ public class CarbonUpdateUtil {
AbsoluteTableIdentifier absoluteTableIdentifier = table.getAbsoluteTableIdentifier();
- CarbonTablePath carbonTablePath = CarbonStorePath
- .getCarbonTablePath(absoluteTableIdentifier.getStorePath(),
- absoluteTableIdentifier.getCarbonTableIdentifier());
-
// read the existing file if present and update the same.
SegmentUpdateDetails[] oldDetails = segmentUpdateStatusManager
.getUpdateStatusDetails();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedRawResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedRawResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedRawResultCollector.java
index 479a684..ea89ce5 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedRawResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedRawResultCollector.java
@@ -200,8 +200,7 @@ public class RestructureBasedRawResultCollector extends RawBasedResultCollector
if (null != defaultValue) {
defaultValueAsLong = ((Integer) defaultValue).longValue();
} else {
- defaultValueAsLong =
- new Integer(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY).longValue();
+ defaultValueAsLong = (long)CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY;
}
keyArrayWithNewAddedColumns[newKeyArrayIndex++] = defaultValueAsLong;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ComplexQueryType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ComplexQueryType.java b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ComplexQueryType.java
index 080d577..2274186 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ComplexQueryType.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ComplexQueryType.java
@@ -20,12 +20,9 @@ package org.apache.carbondata.core.scan.complextypes;
import java.io.IOException;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
-import org.apache.carbondata.core.scan.filter.GenericQueryType;
import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
public class ComplexQueryType {
- protected GenericQueryType children;
-
protected String name;
protected String parentname;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
index f03f6ea..ee04bb6 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
@@ -775,8 +775,7 @@ public class QueryUtil {
new StructQueryType(dimension.getColName(), dimension.getColName(),
dimensionToBlockIndexMap.get(dimension.getOrdinal()));
complexTypeMap.put(dimension.getOrdinal(), parentQueryType);
- parentBlockIndex =
- fillChildrenDetails(eachComplexColumnValueSize, columnIdToDictionaryMap, parentBlockIndex,
+ fillChildrenDetails(eachComplexColumnValueSize, columnIdToDictionaryMap, parentBlockIndex,
dimension, parentQueryType);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
index e035970..aed2775 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
@@ -175,10 +175,10 @@ public class RestructureUtil {
// 2. If default value is not specified then its surrogate key will be
// 1 which is for member default value null
if (isDefaultValueNull(defaultValue)) {
- dictionaryDefaultValue = new Integer(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY);
+ dictionaryDefaultValue = CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY;
} else {
dictionaryDefaultValue =
- new Integer(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY + 1);
+ CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY + 1;
}
return dictionaryDefaultValue;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/scan/expression/RangeExpressionEvaluator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/RangeExpressionEvaluator.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/RangeExpressionEvaluator.java
index 4433e86..d98593c 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/RangeExpressionEvaluator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/RangeExpressionEvaluator.java
@@ -19,6 +19,7 @@ package org.apache.carbondata.core.scan.expression;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -109,9 +110,12 @@ public class RangeExpressionEvaluator {
Map<String, List<FilterModificationNode>> filterExpressionMap) {
List<FilterModificationNode> deleteExp = new ArrayList<>();
- for (String colName : filterExpressionMap.keySet()) {
- // Check is there are multiple list for this Column.
- List<FilterModificationNode> filterExp = filterExpressionMap.get(colName);
+ Iterator<Map.Entry<String, List<FilterModificationNode>>> iterator =
+ filterExpressionMap.entrySet().iterator();
+ Map.Entry<String, List<FilterModificationNode>> nextEntry = null;
+ while (iterator.hasNext()) {
+ nextEntry = iterator.next();
+ List<FilterModificationNode> filterExp = nextEntry.getValue();
if (filterExp.size() > 1) {
// There are multiple Expression for the same column traverse and check if they can
// form a range.
@@ -120,8 +124,8 @@ public class RangeExpressionEvaluator {
for (FilterModificationNode exp : filterExp) {
if ((exp.getExpType() == GREATERTHAN) || (exp.getExpType() == GREATERTHAN_EQUALTO)) {
- if ((null == endMax) || checkLiteralValue(exp.getCurrentExp(),
- endMax.getCurrentExp())) {
+ if ((null == endMax) || ((checkLiteralValue(exp.getCurrentExp(),
+ endMax.getCurrentExp())))) {
if (null == startMin) {
startMin = exp;
} else {
@@ -141,8 +145,8 @@ public class RangeExpressionEvaluator {
}
}
if ((exp.getExpType() == LESSTHAN) || (exp.getExpType() == LESSTHAN_EQUALTO)) {
- if ((null == startMin) || checkLiteralValue(exp.getCurrentExp(),
- startMin.getCurrentExp())) {
+ if ((null == startMin) || ((checkLiteralValue(exp.getCurrentExp(),
+ startMin.getCurrentExp())))) {
if (null == endMax) {
endMax = exp;
} else {
@@ -166,7 +170,7 @@ public class RangeExpressionEvaluator {
if ((null != startMin) && (null != endMax)) {
LOG.info(
"GreaterThan and LessThan Filter Expression changed to Range Expression for column "
- + colName);
+ + nextEntry.getKey());
// the node can be converted to RANGE.
Expression n1 = startMin.getCurrentExp();
Expression n2 = endMax.getCurrentExp();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/ListExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/ListExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/ListExpression.java
index 32ca11c..1c00b14 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/ListExpression.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/ListExpression.java
@@ -48,13 +48,18 @@ public class ListExpression extends Expression {
}
@Override public ExpressionType getFilterExpressionType() {
- // TODO Auto-generated method stub
return ExpressionType.LIST;
}
@Override public String getString() {
- // TODO Auto-generated method stub
- return null;
+ StringBuffer value = new StringBuffer();
+ value.append("ListExpression(");
+ for (Expression expr : children) {
+ value.append(expr.getString()).append(";");
+ }
+ value.append(')');
+
+ return value.toString();
}
@Override public void findAndSetChild(Expression oldExpr, Expression newExpr) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
index 3a19e1b..9bc864d 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
@@ -163,18 +163,17 @@ public class FilterExpressionProcessor implements FilterProcessor {
}
private String joinByteArray(byte[] bytes, char delimiter) {
- String byteArrayAsString = "";
+ StringBuffer byteArrayAsString = new StringBuffer();
+ byteArrayAsString.append("");
if (null != bytes) {
for (int i = 0; i < bytes.length; i++) {
- byteArrayAsString = byteArrayAsString + delimiter + bytes[i];
+ byteArrayAsString.append(delimiter).append(bytes[i]);
}
if (byteArrayAsString.length() > 0) {
- byteArrayAsString = byteArrayAsString.substring(1);
+ return byteArrayAsString.substring(1);
}
- } else {
- byteArrayAsString = null;
}
- return byteArrayAsString;
+ return null;
}
/**
@@ -539,9 +538,9 @@ public class FilterExpressionProcessor implements FilterProcessor {
if ((condExpression.getColumnList().get(0).getCarbonColumn()
.hasEncoding(Encoding.DICTIONARY) && !condExpression.getColumnList().get(0)
.getCarbonColumn().hasEncoding(Encoding.DIRECT_DICTIONARY))
- || (currentCondExpression.getColumnList().get(0).getCarbonColumn().isMeasure())) {
+ || (condExpression.getColumnList().get(0).getCarbonColumn().isMeasure())) {
return new ConditionalFilterResolverImpl(expression, true, true, tableIdentifier,
- currentCondExpression.getColumnList().get(0).getCarbonColumn().isMeasure());
+ condExpression.getColumnList().get(0).getCarbonColumn().isMeasure());
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
index 2ead304..6230771 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
@@ -205,7 +205,7 @@ public final class FilterUtil {
segmentProperties, true);
} else {
return new RestructureIncludeFilterExecutorImpl(dimColResolvedFilterInfo,
- msrColResolvedFilterInfo, segmentProperties, true);
+ msrColResolvedFilterInfo, true);
}
}
if (null != dimColResolvedFilterInfo && dimColResolvedFilterInfo.getDimension().isColumnar()) {
@@ -221,7 +221,7 @@ public final class FilterUtil {
segmentProperties, false);
} else {
return new RestructureIncludeFilterExecutorImpl(dimColResolvedFilterInfo,
- msrColResolvedFilterInfo, segmentProperties, false);
+ msrColResolvedFilterInfo, false);
}
} else {
return new IncludeColGroupFilterExecuterImpl(dimColResolvedFilterInfo, segmentProperties);
@@ -254,7 +254,7 @@ public final class FilterUtil {
segmentProperties, true);
} else {
return new RestructureExcludeFilterExecutorImpl(dimColResolvedFilterInfo,
- msrColResolvedFilterInfo, segmentProperties, true);
+ msrColResolvedFilterInfo, true);
}
}
if ((null != dimColResolvedFilterInfo) && (dimColResolvedFilterInfo.getDimension()
@@ -271,7 +271,7 @@ public final class FilterUtil {
segmentProperties, false);
} else {
return new RestructureExcludeFilterExecutorImpl(dimColResolvedFilterInfo,
- msrColResolvedFilterInfo, segmentProperties, false);
+ msrColResolvedFilterInfo, false);
}
} else {
return new ExcludeColGroupFilterExecuterImpl(dimColResolvedFilterInfo, segmentProperties);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java
index 8e06894..1509949 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java
@@ -19,7 +19,6 @@ package org.apache.carbondata.core.scan.filter.executer;
import java.io.IOException;
import java.util.BitSet;
-import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
@@ -28,10 +27,6 @@ import org.apache.carbondata.core.util.BitSetGroup;
public class RestructureExcludeFilterExecutorImpl extends RestructureEvaluatorImpl {
- protected DimColumnResolvedFilterInfo dimColEvaluatorInfo;
- protected MeasureColumnResolvedFilterInfo measureColumnResolvedFilterInfo;
- protected SegmentProperties segmentProperties;
- protected boolean isMeasure;
/**
* flag to check whether filter values contain the default value applied on the dimension column
@@ -40,12 +35,7 @@ public class RestructureExcludeFilterExecutorImpl extends RestructureEvaluatorIm
protected boolean isDefaultValuePresentInFilterValues;
public RestructureExcludeFilterExecutorImpl(DimColumnResolvedFilterInfo dimColEvaluatorInfo,
- MeasureColumnResolvedFilterInfo measureColumnResolvedFilterInfo,
- SegmentProperties segmentProperties, boolean isMeasure) {
- this.dimColEvaluatorInfo = dimColEvaluatorInfo;
- this.measureColumnResolvedFilterInfo = measureColumnResolvedFilterInfo;
- this.segmentProperties = segmentProperties;
- this.isMeasure = isMeasure;
+ MeasureColumnResolvedFilterInfo measureColumnResolvedFilterInfo, boolean isMeasure) {
if (isMeasure) {
isDefaultValuePresentInFilterValues =
isMeasureDefaultValuePresentInFilterValues(measureColumnResolvedFilterInfo);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java
index 20889ea..0e9a5f0 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java
@@ -19,7 +19,6 @@ package org.apache.carbondata.core.scan.filter.executer;
import java.io.IOException;
import java.util.BitSet;
-import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
@@ -28,11 +27,6 @@ import org.apache.carbondata.core.util.BitSetGroup;
public class RestructureIncludeFilterExecutorImpl extends RestructureEvaluatorImpl {
- protected DimColumnResolvedFilterInfo dimColumnEvaluatorInfo;
- protected MeasureColumnResolvedFilterInfo measureColumnResolvedFilterInfo;
- protected SegmentProperties segmentProperties;
- protected boolean isMeasure;
-
/**
* flag to check whether filter values contain the default value applied on the dimension column
* which does not exist in the current block
@@ -40,12 +34,7 @@ public class RestructureIncludeFilterExecutorImpl extends RestructureEvaluatorIm
protected boolean isDefaultValuePresentInFilterValues;
public RestructureIncludeFilterExecutorImpl(DimColumnResolvedFilterInfo dimColumnEvaluatorInfo,
- MeasureColumnResolvedFilterInfo measureColumnResolvedFilterInfo,
- SegmentProperties segmentProperties, boolean isMeasure) {
- this.dimColumnEvaluatorInfo = dimColumnEvaluatorInfo;
- this.measureColumnResolvedFilterInfo = measureColumnResolvedFilterInfo;
- this.isMeasure = isMeasure;
- this.segmentProperties = segmentProperties;
+ MeasureColumnResolvedFilterInfo measureColumnResolvedFilterInfo, boolean isMeasure) {
if (isMeasure) {
isDefaultValuePresentInFilterValues =
isMeasureDefaultValuePresentInFilterValues(measureColumnResolvedFilterInfo);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
index 40065f1..e066ee3 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
@@ -110,7 +110,7 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
} else {
this.dimColEvaluatorInfoList = dimColEvaluatorInfoList;
}
- if (dimColEvaluatorInfoList.size() > 0) {
+ if (this.dimColEvaluatorInfoList.size() > 0) {
this.isDimensionPresentInCurrentBlock = new boolean[dimColEvaluatorInfoList.size()];
this.dimensionBlocksIndex = new int[dimColEvaluatorInfoList.size()];
} else {
@@ -122,7 +122,7 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
} else {
this.msrColEvalutorInfoList = msrColEvalutorInfoList;
}
- if (msrColEvalutorInfoList.size() > 0) {
+ if (this.msrColEvalutorInfoList.size() > 0) {
this.isMeasurePresentInCurrentBlock = new boolean[msrColEvalutorInfoList.size()];
this.measureBlocksIndex = new int[msrColEvalutorInfoList.size()];
} else {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerImpl.java
index cb9495b..32b1ae1 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerImpl.java
@@ -21,11 +21,9 @@ import org.apache.carbondata.core.scan.expression.Expression;
public class FilterOptimizerImpl implements FilterOptimizer {
- protected FilterOptimizer filterOptimizer;
protected Expression filterExpression;
- public FilterOptimizerImpl(FilterOptimizer filterOptimizer, Expression filterExpression) {
- this.filterOptimizer = filterOptimizer;
+ public FilterOptimizerImpl(Expression filterExpression) {
this.filterExpression = filterExpression;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/scan/filter/optimizer/RangeFilterOptmizer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/optimizer/RangeFilterOptmizer.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/optimizer/RangeFilterOptmizer.java
index 6c61b62..e5cb0ad 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/optimizer/RangeFilterOptmizer.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/optimizer/RangeFilterOptmizer.java
@@ -27,7 +27,7 @@ public class RangeFilterOptmizer extends FilterOptimizerImpl {
RangeExpressionEvaluator rangeExpEvaluator;
public RangeFilterOptmizer(FilterOptimizer filterOptimizer, Expression filterExpression) {
- super(filterOptimizer, filterExpression);
+ super(filterExpression);
this.rangeExpEvaluator = new RangeExpressionEvaluator(filterExpression);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/MeasureColumnResolvedFilterInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/MeasureColumnResolvedFilterInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/MeasureColumnResolvedFilterInfo.java
index 3880eb7..f8586ed 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/MeasureColumnResolvedFilterInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/MeasureColumnResolvedFilterInfo.java
@@ -44,8 +44,6 @@ public class MeasureColumnResolvedFilterInfo extends ColumnResolvedFilterInfo
private boolean isMeasureExistsInCurrentSilce = true;
- private Object defaultValue;
-
private CarbonColumn carbonColumn;
private CarbonMeasure carbonMeasure;
@@ -115,10 +113,6 @@ public class MeasureColumnResolvedFilterInfo extends ColumnResolvedFilterInfo
return true;
}
- public Object getDefaultValue() {
- return defaultValue;
- }
-
public CarbonColumn getCarbonColumn() {
return carbonColumn;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
index b703959..221d243 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
@@ -48,25 +48,28 @@ public class CustomTypeDictionaryVisitor implements ResolvedFilterInfoVisitorInt
public void populateFilterResolvedInfo(ColumnResolvedFilterInfo visitableObj,
FilterResolverMetadata metadata) throws FilterUnsupportedException {
ColumnFilterInfo resolvedFilterObject = null;
- DimColumnResolvedFilterInfo resolveDimension = (DimColumnResolvedFilterInfo) visitableObj;
- List<String> evaluateResultListFinal;
- try {
- evaluateResultListFinal = metadata.getExpression().evaluate(null).getListAsString();
- } catch (FilterIllegalMemberException e) {
- throw new FilterUnsupportedException(e);
+ if (visitableObj instanceof DimColumnResolvedFilterInfo) {
+ DimColumnResolvedFilterInfo resolveDimension = (DimColumnResolvedFilterInfo) visitableObj;
+ List<String> evaluateResultListFinal;
+ try {
+ evaluateResultListFinal = metadata.getExpression().evaluate(null).getListAsString();
+ } catch (FilterIllegalMemberException e) {
+ throw new FilterUnsupportedException(e);
+ }
+ resolvedFilterObject =
+ getDirectDictionaryValKeyMemberForFilter(metadata.getColumnExpression(),
+ evaluateResultListFinal, metadata.isIncludeFilter(),
+ metadata.getColumnExpression().getDimension().getDataType());
+ if (!metadata.isIncludeFilter() && null != resolvedFilterObject && !resolvedFilterObject
+ .getFilterList().contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY)) {
+ // Adding default surrogate key of null member inorder to not display the same while
+ // displaying the report as per hive compatibility.
+ resolvedFilterObject.getFilterList()
+ .add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY);
+ Collections.sort(resolvedFilterObject.getFilterList());
+ }
+ resolveDimension.setFilterValues(resolvedFilterObject);
}
- resolvedFilterObject = getDirectDictionaryValKeyMemberForFilter(metadata.getColumnExpression(),
- evaluateResultListFinal, metadata.isIncludeFilter(),
- metadata.getColumnExpression().getDimension().getDataType());
- if (!metadata.isIncludeFilter() && null != resolvedFilterObject && !resolvedFilterObject
- .getFilterList().contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY)) {
- // Adding default surrogate key of null member inorder to not display the same while
- // displaying the report as per hive compatibility.
- resolvedFilterObject.getFilterList()
- .add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY);
- Collections.sort(resolvedFilterObject.getFilterList());
- }
- resolveDimension.setFilterValues(resolvedFilterObject);
}
protected ColumnFilterInfo getDirectDictionaryValKeyMemberForFilter(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java
index b13b8d3..2fc7503 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java
@@ -45,30 +45,32 @@ public class DictionaryColumnVisitor implements ResolvedFilterInfoVisitorIntf {
public void populateFilterResolvedInfo(ColumnResolvedFilterInfo visitableObj,
FilterResolverMetadata metadata) throws FilterUnsupportedException, IOException {
- DimColumnResolvedFilterInfo resolveDimension = (DimColumnResolvedFilterInfo) visitableObj;
- ColumnFilterInfo resolvedFilterObject = null;
- List<String> evaluateResultListFinal;
- try {
- evaluateResultListFinal = metadata.getExpression().evaluate(null).getListAsString();
- } catch (FilterIllegalMemberException e) {
- throw new FilterUnsupportedException(e);
- }
- resolvedFilterObject = FilterUtil
- .getFilterValues(metadata.getTableIdentifier(), metadata.getColumnExpression(),
- evaluateResultListFinal, metadata.isIncludeFilter());
- if (!metadata.isIncludeFilter() && null != resolvedFilterObject) {
- // Adding default surrogate key of null member inorder to not display the same while
- // displaying the report as per hive compatibility.
- // first check of surrogate key for null value is already added then
- // no need to add again otherwise result will be wrong in case of exclude filter
- // this is because two times it will flip the same bit
- if (!resolvedFilterObject.getFilterList()
- .contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY)) {
- resolvedFilterObject.getFilterList()
- .add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY);
+ if (visitableObj instanceof DimColumnResolvedFilterInfo) {
+ DimColumnResolvedFilterInfo resolveDimension = (DimColumnResolvedFilterInfo) visitableObj;
+ ColumnFilterInfo resolvedFilterObject = null;
+ List<String> evaluateResultListFinal;
+ try {
+ evaluateResultListFinal = metadata.getExpression().evaluate(null).getListAsString();
+ } catch (FilterIllegalMemberException e) {
+ throw new FilterUnsupportedException(e);
+ }
+ resolvedFilterObject = FilterUtil
+ .getFilterValues(metadata.getTableIdentifier(), metadata.getColumnExpression(),
+ evaluateResultListFinal, metadata.isIncludeFilter());
+ if (!metadata.isIncludeFilter() && null != resolvedFilterObject) {
+ // Adding default surrogate key of null member inorder to not display the same while
+ // displaying the report as per hive compatibility.
+ // first check of surrogate key for null value is already added then
+ // no need to add again otherwise result will be wrong in case of exclude filter
+ // this is because two times it will flip the same bit
+ if (!resolvedFilterObject.getFilterList()
+ .contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY)) {
+ resolvedFilterObject.getFilterList()
+ .add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY);
+ }
+ Collections.sort(resolvedFilterObject.getFilterList());
}
- Collections.sort(resolvedFilterObject.getFilterList());
+ resolveDimension.setFilterValues(resolvedFilterObject);
}
- resolveDimension.setFilterValues(resolvedFilterObject);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/MeasureColumnVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/MeasureColumnVisitor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/MeasureColumnVisitor.java
index 8c86a2b..ac4ced8 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/MeasureColumnVisitor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/MeasureColumnVisitor.java
@@ -45,33 +45,35 @@ public class MeasureColumnVisitor implements ResolvedFilterInfoVisitorIntf {
*/
public void populateFilterResolvedInfo(ColumnResolvedFilterInfo visitableObj,
FilterResolverMetadata metadata) throws FilterUnsupportedException {
- MeasureColumnResolvedFilterInfo resolveDimension =
- (MeasureColumnResolvedFilterInfo) visitableObj;
- ColumnFilterInfo resolvedFilterObject = null;
- List<String> evaluateResultListFinal = null;
- try {
- // handling for is null case scenarios
- if (metadata.getExpression() instanceof EqualToExpression) {
- EqualToExpression expression = (EqualToExpression) metadata.getExpression();
- if (expression.isNull) {
- evaluateResultListFinal = new ArrayList<>(1);
+ if (visitableObj instanceof MeasureColumnResolvedFilterInfo) {
+ MeasureColumnResolvedFilterInfo resolveDimension =
+ (MeasureColumnResolvedFilterInfo) visitableObj;
+ ColumnFilterInfo resolvedFilterObject = null;
+ List<String> evaluateResultListFinal = new ArrayList<>(1);
+ try {
+ // handling for is null case scenarios
+ if (metadata.getExpression() instanceof EqualToExpression) {
+ EqualToExpression expression = (EqualToExpression) metadata.getExpression();
+ if (expression.isNull) {
+ evaluateResultListFinal = new ArrayList<>(1);
+ evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
+ }
+ } else {
+ evaluateResultListFinal = metadata.getExpression().evaluate(null).getListAsString();
+ }
+ // Adding default null member inorder to not display the same while
+ // displaying the report as per hive compatibility.
+ if (!metadata.isIncludeFilter() && !evaluateResultListFinal
+ .contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
}
- } else {
- evaluateResultListFinal = metadata.getExpression().evaluate(null).getListAsString();
- }
- // Adding default null member inorder to not display the same while
- // displaying the report as per hive compatibility.
- if (!metadata.isIncludeFilter() && !evaluateResultListFinal
- .contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
- evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
+ } catch (FilterIllegalMemberException e) {
+ throw new FilterUnsupportedException(e);
}
- } catch (FilterIllegalMemberException e) {
- throw new FilterUnsupportedException(e);
+ resolvedFilterObject = FilterUtil
+ .getMeasureValKeyMemberForFilter(evaluateResultListFinal, metadata.isIncludeFilter(),
+ metadata.getColumnExpression().getDataType(), resolveDimension.getMeasure());
+ resolveDimension.setFilterValues(resolvedFilterObject);
}
- resolvedFilterObject = FilterUtil
- .getMeasureValKeyMemberForFilter(evaluateResultListFinal, metadata.isIncludeFilter(),
- metadata.getColumnExpression().getDataType(), resolveDimension.getMeasure());
- resolveDimension.setFilterValues(resolvedFilterObject);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java
index a8af72e..132f854 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java
@@ -45,32 +45,34 @@ public class NoDictionaryTypeVisitor implements ResolvedFilterInfoVisitorIntf {
*/
public void populateFilterResolvedInfo(ColumnResolvedFilterInfo visitableObj,
FilterResolverMetadata metadata) throws FilterUnsupportedException {
- DimColumnResolvedFilterInfo resolveDimension = (DimColumnResolvedFilterInfo) visitableObj;
- ColumnFilterInfo resolvedFilterObject = null;
- List<String> evaluateResultListFinal = null;
- try {
- // handling for is null case scenarios
- if (metadata.getExpression() instanceof EqualToExpression) {
- EqualToExpression expression = (EqualToExpression) metadata.getExpression();
- if (expression.isNull) {
- evaluateResultListFinal = new ArrayList<>(1);
+ if (visitableObj instanceof DimColumnResolvedFilterInfo) {
+ DimColumnResolvedFilterInfo resolveDimension = (DimColumnResolvedFilterInfo) visitableObj;
+ ColumnFilterInfo resolvedFilterObject = null;
+ List<String> evaluateResultListFinal = new ArrayList<>(1);
+ try {
+ // handling for is null case scenarios
+ if (metadata.getExpression() instanceof EqualToExpression) {
+ EqualToExpression expression = (EqualToExpression) metadata.getExpression();
+ if (expression.isNull) {
+ evaluateResultListFinal = new ArrayList<>(1);
+ evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
+ }
+ } else {
+ evaluateResultListFinal = metadata.getExpression().evaluate(null).getListAsString();
+ }
+ // Adding default null member inorder to not display the same while
+ // displaying the report as per hive compatibility.
+ if (!metadata.isIncludeFilter() && !evaluateResultListFinal
+ .contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
}
- } else {
- evaluateResultListFinal = metadata.getExpression().evaluate(null).getListAsString();
- }
- // Adding default null member inorder to not display the same while
- // displaying the report as per hive compatibility.
- if (!metadata.isIncludeFilter() && !evaluateResultListFinal
- .contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
- evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
+ } catch (FilterIllegalMemberException e) {
+ throw new FilterUnsupportedException(e);
}
- } catch (FilterIllegalMemberException e) {
- throw new FilterUnsupportedException(e);
+ resolvedFilterObject = FilterUtil
+ .getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, metadata.isIncludeFilter(),
+ metadata.getColumnExpression().getDataType());
+ resolveDimension.setFilterValues(resolvedFilterObject);
}
- resolvedFilterObject = FilterUtil
- .getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, metadata.isIncludeFilter(),
- metadata.getColumnExpression().getDataType());
- resolveDimension.setFilterValues(resolvedFilterObject);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDictionaryColumnVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDictionaryColumnVisitor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDictionaryColumnVisitor.java
index 41c95e5..3d16cd2 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDictionaryColumnVisitor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDictionaryColumnVisitor.java
@@ -44,20 +44,22 @@ public class RangeDictionaryColumnVisitor extends DictionaryColumnVisitor
*/
public void populateFilterResolvedInfo(ColumnResolvedFilterInfo visitableObj,
FilterResolverMetadata metadata) throws FilterUnsupportedException, IOException {
- DimColumnResolvedFilterInfo resolveDimension = (DimColumnResolvedFilterInfo) visitableObj;
- ColumnFilterInfo resolvedFilterObject = null;
- List<String> evaluateResultListFinal;
- resolvedFilterObject = FilterUtil
- .getFilterListForAllValues(metadata.getTableIdentifier(), metadata.getExpression(),
- metadata.getColumnExpression(), metadata.isIncludeFilter());
+ if (visitableObj instanceof DimColumnResolvedFilterInfo) {
+ DimColumnResolvedFilterInfo resolveDimension = (DimColumnResolvedFilterInfo) visitableObj;
+ ColumnFilterInfo resolvedFilterObject = null;
+ List<String> evaluateResultListFinal;
+ resolvedFilterObject = FilterUtil
+ .getFilterListForAllValues(metadata.getTableIdentifier(), metadata.getExpression(),
+ metadata.getColumnExpression(), metadata.isIncludeFilter());
- if (!metadata.isIncludeFilter() && null != resolvedFilterObject) {
- // Adding default surrogate key of null member inorder to not display the same while
- // displaying the report as per hive compatibility.
- resolvedFilterObject.getFilterList()
- .add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY);
- Collections.sort(resolvedFilterObject.getFilterList());
+ if (!metadata.isIncludeFilter() && null != resolvedFilterObject) {
+ // Adding default surrogate key of null member inorder to not display the same while
+ // displaying the report as per hive compatibility.
+ resolvedFilterObject.getFilterList()
+ .add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY);
+ Collections.sort(resolvedFilterObject.getFilterList());
+ }
+ resolveDimension.setFilterValues(resolvedFilterObject);
}
- resolveDimension.setFilterValues(resolvedFilterObject);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDirectDictionaryVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDirectDictionaryVisitor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDirectDictionaryVisitor.java
index d740648..918fd9e 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDirectDictionaryVisitor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDirectDictionaryVisitor.java
@@ -46,36 +46,39 @@ public class RangeDirectDictionaryVisitor extends CustomTypeDictionaryVisitor
*/
public void populateFilterResolvedInfo(ColumnResolvedFilterInfo visitableObj,
FilterResolverMetadata metadata) throws FilterUnsupportedException {
- DimColumnResolvedFilterInfo resolveDimension = (DimColumnResolvedFilterInfo) visitableObj;
- ColumnFilterInfo resolvedFilterObject = null;
- List<ExpressionResult> listOfExpressionResults = new ArrayList<ExpressionResult>(20);
- List<String> evaluateResultListFinal = new ArrayList<String>();
- try {
- listOfExpressionResults = ((RangeExpression) metadata.getExpression()).getLiterals();
+ if (visitableObj instanceof DimColumnResolvedFilterInfo) {
+ DimColumnResolvedFilterInfo resolveDimension = (DimColumnResolvedFilterInfo) visitableObj;
+ ColumnFilterInfo resolvedFilterObject = null;
+ List<ExpressionResult> listOfExpressionResults = null;
+ List<String> evaluateResultListFinal = new ArrayList<String>();
+ try {
+ listOfExpressionResults = ((RangeExpression) metadata.getExpression()).getLiterals();
- for (ExpressionResult result : listOfExpressionResults) {
- if (result.getString() == null) {
- evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
- continue;
+ for (ExpressionResult result : listOfExpressionResults) {
+ if (result.getString() == null) {
+ evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
+ continue;
+ }
+ evaluateResultListFinal.add(result.getString());
}
- evaluateResultListFinal.add(result.getString());
+ } catch (FilterIllegalMemberException e) {
+ throw new FilterUnsupportedException(e);
}
- } catch (FilterIllegalMemberException e) {
- throw new FilterUnsupportedException(e);
- }
- resolvedFilterObject = getDirectDictionaryValKeyMemberForFilter(metadata.getColumnExpression(),
- evaluateResultListFinal, metadata.isIncludeFilter(),
- metadata.getColumnExpression().getDimension().getDataType());
+ resolvedFilterObject =
+ getDirectDictionaryValKeyMemberForFilter(metadata.getColumnExpression(),
+ evaluateResultListFinal, metadata.isIncludeFilter(),
+ metadata.getColumnExpression().getDimension().getDataType());
- if (!metadata.isIncludeFilter() && null != resolvedFilterObject && !resolvedFilterObject
- .getFilterList().contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY)) {
- // Adding default surrogate key of null member inorder to not display the same while
- // displaying the report as per hive compatibility.
- resolvedFilterObject.getFilterList()
- .add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY);
- Collections.sort(resolvedFilterObject.getFilterList());
+ if (!metadata.isIncludeFilter() && null != resolvedFilterObject && !resolvedFilterObject
+ .getFilterList().contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY)) {
+ // Adding default surrogate key of null member inorder to not display the same while
+ // displaying the report as per hive compatibility.
+ resolvedFilterObject.getFilterList()
+ .add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY);
+ Collections.sort(resolvedFilterObject.getFilterList());
+ }
+ resolveDimension.setFilterValues(resolvedFilterObject);
}
- resolveDimension.setFilterValues(resolvedFilterObject);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeNoDictionaryTypeVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeNoDictionaryTypeVisitor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeNoDictionaryTypeVisitor.java
index 87864f7..d4c44ba 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeNoDictionaryTypeVisitor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeNoDictionaryTypeVisitor.java
@@ -47,35 +47,37 @@ public class RangeNoDictionaryTypeVisitor extends NoDictionaryTypeVisitor
*/
public void populateFilterResolvedInfo(ColumnResolvedFilterInfo visitableObj,
FilterResolverMetadata metadata) throws FilterUnsupportedException {
- DimColumnResolvedFilterInfo resolveDimension = (DimColumnResolvedFilterInfo) visitableObj;
- ColumnFilterInfo resolvedFilterObject = null;
- List<ExpressionResult> listOfExpressionResults = new ArrayList<ExpressionResult>(20);
- List<String> evaluateResultListFinal = new ArrayList<String>();
- try {
- // Add The Range Filter Values.
- if (metadata.getExpression() instanceof RangeExpression) {
- listOfExpressionResults = ((RangeExpression) metadata.getExpression()).getLiterals();
- }
+ if (visitableObj instanceof DimColumnResolvedFilterInfo) {
+ DimColumnResolvedFilterInfo resolveDimension = (DimColumnResolvedFilterInfo) visitableObj;
+ ColumnFilterInfo resolvedFilterObject = null;
+ List<ExpressionResult> listOfExpressionResults = new ArrayList<ExpressionResult>(20);
+ List<String> evaluateResultListFinal = new ArrayList<String>();
+ try {
+ // Add The Range Filter Values.
+ if (metadata.getExpression() instanceof RangeExpression) {
+ listOfExpressionResults = ((RangeExpression) metadata.getExpression()).getLiterals();
+ }
- for (ExpressionResult result : listOfExpressionResults) {
- if (result.getString() == null) {
- evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
- continue;
+ for (ExpressionResult result : listOfExpressionResults) {
+ if (result.getString() == null) {
+ evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
+ continue;
+ }
+ evaluateResultListFinal.add(result.getString());
}
- evaluateResultListFinal.add(result.getString());
- }
- // evaluateResultListFinal.add(metadata.getExpression().evaluate().getListAsString());
- if (!metadata.isIncludeFilter() && !evaluateResultListFinal
- .contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
- evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
+ // evaluateResultListFinal.add(metadata.getExpression().evaluate().getListAsString());
+ if (!metadata.isIncludeFilter() && !evaluateResultListFinal
+ .contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
+ evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
+ }
+ } catch (FilterIllegalMemberException e) {
+ throw new FilterUnsupportedException(e);
}
- } catch (FilterIllegalMemberException e) {
- throw new FilterUnsupportedException(e);
+ resolvedFilterObject = FilterUtil
+ .getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, metadata.isIncludeFilter(),
+ metadata.getColumnExpression().getDataType());
+ resolveDimension.setFilterValues(resolvedFilterObject);
}
- resolvedFilterObject = FilterUtil
- .getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, metadata.isIncludeFilter(),
- metadata.getColumnExpression().getDataType());
- resolveDimension.setFilterValues(resolvedFilterObject);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/scan/result/vector/ColumnVectorInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/ColumnVectorInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/ColumnVectorInfo.java
index 631ab97..a5f81b9 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/ColumnVectorInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/ColumnVectorInfo.java
@@ -36,4 +36,27 @@ public class ColumnVectorInfo implements Comparable<ColumnVectorInfo> {
@Override public int compareTo(ColumnVectorInfo o) {
return ordinal - o.ordinal;
}
+
+ @Override public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+
+ if (null == obj) {
+ return false;
+ }
+
+ if (!(obj instanceof ColumnVectorInfo)) {
+ return false;
+ }
+
+ return ordinal == ((ColumnVectorInfo) obj).ordinal;
+ }
+
+ @Override public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + (ordinal);
+ return result;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsRecorderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsRecorderImpl.java b/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsRecorderImpl.java
index ffb7d7f..9807eaa 100644
--- a/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsRecorderImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsRecorderImpl.java
@@ -164,14 +164,16 @@ public class QueryStatisticsRecorderImpl implements QueryStatisticsRecorder, Ser
values.add(String.valueOf(result_size));
StringBuilder tableInfo = new StringBuilder();
String[] columns = headers.split(",");
- String line = "";
- String hearLine = "";
- String valueLine = "";
+ StringBuilder line = new StringBuilder("");
+ StringBuilder hearLine = new StringBuilder("");
+ StringBuilder valueLine = new StringBuilder("");
for (int i = 0; i < columns.length; i++) {
int len = Math.max(columns[i].length(), values.get(i).length());
- line += "+" + printLine("-", len);
- hearLine += "|" + printLine(" ", len - columns[i].length()) + columns[i];
- valueLine += "|" + printLine(" ", len - values.get(i).length()) + values.get(i);
+ line.append("+").append(printLine("-", len));
+ hearLine.append("|").append(printLine(" ", len - columns[i].length()))
+ .append(columns[i]);
+ valueLine.append("|").append(printLine(" ", len - values.get(i).length()))
+ .append(values.get(i));
}
// struct table info
tableInfo.append(line).append("+").append("\n");
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
index 831a2fb..d1c16bb 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
@@ -143,8 +143,8 @@ public final class ByteUtil {
* 32-bit. On the other hand, it is substantially faster on 64-bit.
*/
for (int i = 0; i < minWords * SIZEOF_LONG; i += SIZEOF_LONG) {
- long lw = CarbonUnsafe.unsafe.getLong(buffer1, offset1Adj + (long) i);
- long rw = CarbonUnsafe.unsafe.getLong(buffer2, offset2Adj + (long) i);
+ long lw = CarbonUnsafe.getUnsafe().getLong(buffer1, offset1Adj + (long) i);
+ long rw = CarbonUnsafe.getUnsafe().getLong(buffer2, offset2Adj + (long) i);
long diff = lw ^ rw;
if (diff != 0) {
@@ -209,8 +209,10 @@ public final class ByteUtil {
if (minLength > 7) {
minWords = minLength / SIZEOF_LONG;
for (int i = 0; i < minWords * SIZEOF_LONG; i += SIZEOF_LONG) {
- long lw = CarbonUnsafe.unsafe.getLong(buffer1, CarbonUnsafe.BYTE_ARRAY_OFFSET + (long) i);
- long rw = CarbonUnsafe.unsafe.getLong(buffer2, CarbonUnsafe.BYTE_ARRAY_OFFSET + (long) i);
+ long lw =
+ CarbonUnsafe.getUnsafe().getLong(buffer1, CarbonUnsafe.BYTE_ARRAY_OFFSET + (long) i);
+ long rw =
+ CarbonUnsafe.getUnsafe().getLong(buffer2, CarbonUnsafe.BYTE_ARRAY_OFFSET + (long) i);
long diff = lw ^ rw;
if (diff != 0) {
@@ -252,6 +254,7 @@ public final class ByteUtil {
}
return len1 - len2;
}
+
public boolean equals(byte[] buffer1, byte[] buffer2) {
if (buffer1.length != buffer2.length) {
return false;
@@ -259,8 +262,8 @@ public final class ByteUtil {
int len = buffer1.length / 8;
long currentOffset = CarbonUnsafe.BYTE_ARRAY_OFFSET;
for (int i = 0; i < len; i++) {
- long lw = CarbonUnsafe.unsafe.getLong(buffer1, currentOffset);
- long rw = CarbonUnsafe.unsafe.getLong(buffer2, currentOffset);
+ long lw = CarbonUnsafe.getUnsafe().getLong(buffer1, currentOffset);
+ long rw = CarbonUnsafe.getUnsafe().getLong(buffer2, currentOffset);
if (lw != rw) {
return false;
}
@@ -269,8 +272,8 @@ public final class ByteUtil {
len = buffer1.length % 8;
if (len > 0) {
for (int i = 0; i < len; i += 1) {
- long lw = CarbonUnsafe.unsafe.getByte(buffer1, currentOffset);
- long rw = CarbonUnsafe.unsafe.getByte(buffer2, currentOffset);
+ long lw = CarbonUnsafe.getUnsafe().getByte(buffer1, currentOffset);
+ long rw = CarbonUnsafe.getUnsafe().getByte(buffer2, currentOffset);
if (lw != rw) {
return false;
}
@@ -288,8 +291,8 @@ public final class ByteUtil {
int len = length1 / 8;
long currentOffset = CarbonUnsafe.BYTE_ARRAY_OFFSET;
for (int i = 0; i < len; i++) {
- long lw = CarbonUnsafe.unsafe.getLong(buffer1, currentOffset + offset1);
- long rw = CarbonUnsafe.unsafe.getLong(buffer2, currentOffset + offset2);
+ long lw = CarbonUnsafe.getUnsafe().getLong(buffer1, currentOffset + offset1);
+ long rw = CarbonUnsafe.getUnsafe().getLong(buffer2, currentOffset + offset2);
if (lw != rw) {
return false;
}
@@ -298,8 +301,8 @@ public final class ByteUtil {
len = buffer1.length % 8;
if (len > 0) {
for (int i = 0; i < len; i += 1) {
- long lw = CarbonUnsafe.unsafe.getByte(buffer1, currentOffset + offset1);
- long rw = CarbonUnsafe.unsafe.getByte(buffer2, currentOffset + offset2);
+ long lw = CarbonUnsafe.getUnsafe().getByte(buffer1, currentOffset + offset1);
+ long rw = CarbonUnsafe.getUnsafe().getByte(buffer2, currentOffset + offset2);
if (lw != rw) {
return false;
}
@@ -426,12 +429,12 @@ public final class ByteUtil {
throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_SHORT);
}
short n = 0;
- if (CarbonUnsafe.unsafe != null) {
+ if (CarbonUnsafe.getUnsafe() != null) {
if (CarbonUnsafe.ISLITTLEENDIAN) {
n = Short.reverseBytes(
- CarbonUnsafe.unsafe.getShort(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET));
+ CarbonUnsafe.getUnsafe().getShort(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET));
} else {
- n = CarbonUnsafe.unsafe.getShort(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
+ n = CarbonUnsafe.getUnsafe().getShort(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
}
} else {
@@ -497,12 +500,12 @@ public final class ByteUtil {
throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_INT);
}
int n = 0;
- if (CarbonUnsafe.unsafe != null) {
+ if (CarbonUnsafe.getUnsafe() != null) {
if (CarbonUnsafe.ISLITTLEENDIAN) {
n = Integer.reverseBytes(
- CarbonUnsafe.unsafe.getInt(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET));
+ CarbonUnsafe.getUnsafe().getInt(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET));
} else {
- n = CarbonUnsafe.unsafe.getInt(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
+ n = CarbonUnsafe.getUnsafe().getInt(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
}
} else {
for (int i = offset; i < (offset + length); i++) {
@@ -550,12 +553,12 @@ public final class ByteUtil {
throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_LONG);
}
long l = 0;
- if (CarbonUnsafe.unsafe != null) {
+ if (CarbonUnsafe.getUnsafe() != null) {
if (CarbonUnsafe.ISLITTLEENDIAN) {
l = Long.reverseBytes(
- CarbonUnsafe.unsafe.getLong(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET));
+ CarbonUnsafe.getUnsafe().getLong(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET));
} else {
- l = CarbonUnsafe.unsafe.getLong(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
+ l = CarbonUnsafe.getUnsafe().getLong(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
}
} else {
for (int i = offset; i < offset + length; i++) {
@@ -594,11 +597,11 @@ public final class ByteUtil {
"Not enough room to put an int at" + " offset " + offset + " in a " + bytes.length
+ " byte array");
}
- if (CarbonUnsafe.unsafe != null) {
+ if (CarbonUnsafe.getUnsafe() != null) {
if (CarbonUnsafe.ISLITTLEENDIAN) {
val = Integer.reverseBytes(val);
}
- CarbonUnsafe.unsafe.putInt(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET, val);
+ CarbonUnsafe.getUnsafe().putInt(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET, val);
return offset + ByteUtil.SIZEOF_INT;
} else {
for (int i = offset + 3; i > offset; i--) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
index 669d3f2..061721b 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
@@ -137,22 +137,18 @@ public final class CarbonProperties {
}
private void validateLockType() {
- String lockTypeConfigured = carbonProperties.getProperty(CarbonCommonConstants.LOCK_TYPE);
- if (null != lockTypeConfigured) {
- lockTypeConfigured = lockTypeConfigured.toUpperCase();
- switch (lockTypeConfigured) {
- // if user is setting the lock type as CARBON_LOCK_TYPE_ZOOKEEPER then no need to validate
- // else validate based on the file system type for LOCAL file system lock will be
- // CARBON_LOCK_TYPE_LOCAL and for the distributed one CARBON_LOCK_TYPE_HDFS
- case CarbonCommonConstants.CARBON_LOCK_TYPE_ZOOKEEPER:
- break;
- case CarbonCommonConstants.CARBON_LOCK_TYPE_LOCAL:
- case CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS:
- default:
- validateAndConfigureLockType(lockTypeConfigured);
- }
- } else {
- validateAndConfigureLockType(lockTypeConfigured);
+ String lockTypeConfigured = carbonProperties
+ .getProperty(CarbonCommonConstants.LOCK_TYPE, CarbonCommonConstants.LOCK_TYPE_DEFAULT);
+ switch (lockTypeConfigured.toUpperCase()) {
+ // if user is setting the lock type as CARBON_LOCK_TYPE_ZOOKEEPER then no need to validate
+ // else validate based on the file system type for LOCAL file system lock will be
+ // CARBON_LOCK_TYPE_LOCAL and for the distributed one CARBON_LOCK_TYPE_HDFS
+ case CarbonCommonConstants.CARBON_LOCK_TYPE_ZOOKEEPER:
+ break;
+ case CarbonCommonConstants.CARBON_LOCK_TYPE_LOCAL:
+ case CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS:
+ default:
+ validateAndConfigureLockType(lockTypeConfigured);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 15512a8..3bf837b 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -304,8 +304,9 @@ public final class CarbonUtil {
*/
private static void deleteRecursive(File f) throws IOException {
if (f.isDirectory()) {
- if (f.listFiles() != null) {
- for (File c : f.listFiles()) {
+ File[] files = f.listFiles();
+ if (null != files) {
+ for (File c : files) {
deleteRecursive(c);
}
}
@@ -950,6 +951,7 @@ public final class CarbonUtil {
fileReader.finish();
} catch (IOException e) {
// ignore the exception as nothing we can do about it
+ fileReader = null;
}
}
}
@@ -1438,7 +1440,8 @@ public final class CarbonUtil {
return valueEncoderMeta;
}
- public static byte[] serializeEncodeMetaUsingByteBuffer(ValueEncoderMeta valueEncoderMeta) {
+ public static byte[] serializeEncodeMetaUsingByteBuffer(ValueEncoderMeta valueEncoderMeta)
+ throws IOException {
ByteBuffer buffer = null;
switch (valueEncoderMeta.getType()) {
case LONG:
@@ -1463,6 +1466,8 @@ public final class CarbonUtil {
buffer = ByteBuffer.allocate(CarbonCommonConstants.INT_SIZE_IN_BYTE + 3);
buffer.putChar(valueEncoderMeta.getTypeInChar());
break;
+ default:
+ throw new IOException("Unsupported datatype: " + valueEncoderMeta.getType());
}
buffer.putInt(0); // decimal point, not used
buffer.put(valueEncoderMeta.getDataTypeSelected());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/util/DataTypeConverterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeConverterImpl.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeConverterImpl.java
index c71a4d0..afc81bf 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeConverterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeConverterImpl.java
@@ -35,7 +35,7 @@ public class DataTypeConverterImpl implements DataTypeConverter, Serializable {
}
public byte[] convertFromStringToByte(Object data) {
- return data.toString().getBytes();
+ return data.toString().getBytes(CarbonCommonConstants.DEFAULT_CHARSET_CLASS);
}
public Object convertFromStringToUTF8String(Object data) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
index 6e54f1f..2cd7ce5 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -154,7 +154,11 @@ public final class DataTypeUtil {
bigDecimalMsrValue =
bigDecimalMsrValue.setScale(carbonMeasure.getScale(), RoundingMode.HALF_UP);
}
- return normalizeDecimalValue(bigDecimalMsrValue, carbonMeasure.getPrecision());
+ if (null != bigDecimalMsrValue) {
+ return normalizeDecimalValue(bigDecimalMsrValue, carbonMeasure.getPrecision());
+ } else {
+ return bigDecimalMsrValue;
+ }
default:
return measurePage.getDouble(index);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/util/comparator/Comparator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/comparator/Comparator.java b/core/src/main/java/org/apache/carbondata/core/util/comparator/Comparator.java
index 1f43c7c..5681364 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/comparator/Comparator.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/comparator/Comparator.java
@@ -120,13 +120,7 @@ class DoubleSerializableComparator implements SerializableComparator {
} else if (key2 == null) {
return 1;
}
- if ((double) key1 < (double) key2) {
- return -1;
- } else if ((double) key1 > (double) key2) {
- return 1;
- } else {
- return 0;
- }
+ return ((Double)key1).compareTo((Double)key2);
}
}