You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by gv...@apache.org on 2017/04/20 12:32:36 UTC
[1/2] incubator-carbondata git commit: Fix Unsafe merge sort issue
Repository: incubator-carbondata
Updated Branches:
refs/heads/master ca07119d1 -> 278ed4d54
Fix Unsafe merge sort issue
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/bfa306b8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/bfa306b8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/bfa306b8
Branch: refs/heads/master
Commit: bfa306b8cc2bad6fa4e45d51c54723f11b0732d1
Parents: ca07119
Author: ravipesala <ra...@gmail.com>
Authored: Thu Apr 20 15:34:20 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Apr 20 15:34:20 2017 +0530
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 2 +-
.../apache/carbondata/core/util/ByteUtil.java | 63 --------------------
.../unsafe/comparator/UnsafeRowComparator.java | 34 ++++++-----
.../holder/UnsafeInmemoryMergeHolder.java | 6 +-
4 files changed, 26 insertions(+), 79 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/bfa306b8/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index b82d53c..ee18321 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -1081,7 +1081,7 @@ public final class CarbonCommonConstants {
public static final String ENABLE_INMEMORY_MERGE_SORT = "enable.inmemory.merge.sort";
- public static final String ENABLE_INMEMORY_MERGE_SORT_DEFAULT = "true";
+ public static final String ENABLE_INMEMORY_MERGE_SORT_DEFAULT = "false";
public static final String OFFHEAP_SORT_CHUNK_SIZE_IN_MB = "offheap.sort.chunk.size.inmb";
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/bfa306b8/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
index 8f83f3d..7e7e01b 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
@@ -248,69 +248,6 @@ public final class ByteUtil {
return len1 - len2;
}
- public int compareUnsafeTo(Object baseObject1, Object baseObject2, long address1, long address2,
- int len1, int len2, int minLength) {
-
- int minWords = 0;
-
- /*
- * Compare 8 bytes at a time. Benchmarking shows comparing 8 bytes
- * at a time is no slower than comparing 4 bytes at a time even on
- * 32-bit. On the other hand, it is substantially faster on 64-bit.
- */
- if (minLength > 7) {
- minWords = minLength / SIZEOF_LONG;
- for (int i = 0; i < minWords * SIZEOF_LONG; i += SIZEOF_LONG) {
- long lw = CarbonUnsafe.unsafe
- .getLong(baseObject1, CarbonUnsafe.BYTE_ARRAY_OFFSET + (long) i + address1);
- long rw = CarbonUnsafe.unsafe
- .getLong(baseObject2, CarbonUnsafe.BYTE_ARRAY_OFFSET + (long) i + address2);
- long diff = lw ^ rw;
-
- if (diff != 0) {
- if (!CarbonUnsafe.ISLITTLEENDIAN) {
- return lessThanUnsigned(lw, rw) ? -1 : 1;
- }
-
- // Use binary search
- int k = 0;
- int y;
- int x = (int) diff;
- if (x == 0) {
- x = (int) (diff >>> 32);
- k = 32;
- }
- y = x << 16;
- if (y == 0) {
- k += 16;
- } else {
- x = y;
- }
-
- y = x << 8;
- if (y == 0) {
- k += 8;
- }
- return (int) (((lw >>> k) & 0xFFL) - ((rw >>> k) & 0xFFL));
- }
- }
- }
-
- // The epilogue to cover the last (minLength % 8) elements.
- for (int i = minWords * SIZEOF_LONG; i < minLength; i++) {
- int a =
- (CarbonUnsafe.unsafe.getByte(baseObject1, CarbonUnsafe.BYTE_ARRAY_OFFSET + i + address1)
- & 0xff);
- int b =
- (CarbonUnsafe.unsafe.getByte(baseObject2, CarbonUnsafe.BYTE_ARRAY_OFFSET + i + address2)
- & 0xff);
- if (a != b) {
- return a - b;
- }
- }
- return len1 - len2;
- }
-
public boolean equals(byte[] buffer1, byte[] buffer2) {
if (buffer1.length != buffer2.length) {
return false;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/bfa306b8/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/comparator/UnsafeRowComparator.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/comparator/UnsafeRowComparator.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/comparator/UnsafeRowComparator.java
index e61a284..8f048bd 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/comparator/UnsafeRowComparator.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/comparator/UnsafeRowComparator.java
@@ -50,20 +50,23 @@ public class UnsafeRowComparator implements Comparator<UnsafeCarbonRow> {
for (boolean isNoDictionary : noDictionaryColMaping) {
if (isNoDictionary) {
short aShort1 = CarbonUnsafe.unsafe.getShort(baseObject, rowA + sizeA);
+ byte[] byteArr1 = new byte[aShort1];
sizeA += 2;
+ CarbonUnsafe.unsafe.copyMemory(baseObject, rowA + sizeA, byteArr1,
+ CarbonUnsafe.BYTE_ARRAY_OFFSET, aShort1);
+ sizeA += aShort1;
short aShort2 = CarbonUnsafe.unsafe.getShort(baseObject, rowB + sizeB);
+ byte[] byteArr2 = new byte[aShort2];
sizeB += 2;
- int minLength = (aShort1 <= aShort2) ? aShort1 : aShort2;
- int difference = UnsafeComparer.INSTANCE
- .compareUnsafeTo(baseObject, baseObject, rowA + sizeA, rowB + sizeB, aShort1, aShort2,
- minLength);
+ CarbonUnsafe.unsafe.copyMemory(baseObject, rowB + sizeB, byteArr2,
+ CarbonUnsafe.BYTE_ARRAY_OFFSET, aShort2);
+ sizeB += aShort2;
+ int difference = UnsafeComparer.INSTANCE.compareTo(byteArr1, byteArr2);
if (difference != 0) {
return difference;
}
- sizeA += aShort1;
- sizeB += aShort2;
} else {
int dimFieldA = CarbonUnsafe.unsafe.getInt(baseObject, rowA + sizeA);
sizeA += 4;
@@ -92,22 +95,25 @@ public class UnsafeRowComparator implements Comparator<UnsafeCarbonRow> {
for (boolean isNoDictionary : noDictionaryColMaping) {
if (isNoDictionary) {
short aShort1 = CarbonUnsafe.unsafe.getShort(baseObjectL, rowA + sizeA);
+ byte[] byteArr1 = new byte[aShort1];
sizeA += 2;
+ CarbonUnsafe.unsafe
+ .copyMemory(baseObjectL, rowA + sizeA, byteArr1, CarbonUnsafe.BYTE_ARRAY_OFFSET,
+ aShort1);
+ sizeA += aShort1;
short aShort2 = CarbonUnsafe.unsafe.getShort(baseObjectR, rowB + sizeB);
+ byte[] byteArr2 = new byte[aShort2];
sizeB += 2;
+ CarbonUnsafe.unsafe
+ .copyMemory(baseObjectR, rowB + sizeB, byteArr2, CarbonUnsafe.BYTE_ARRAY_OFFSET,
+ aShort2);
+ sizeB += aShort2;
- int minLength = (aShort1 <= aShort2) ? aShort1 : aShort2;
-
- int difference = UnsafeComparer.INSTANCE
- .compareUnsafeTo(baseObjectL, baseObjectR, rowA + sizeA, rowB + sizeB, aShort1, aShort2,
- minLength);
-
+ int difference = UnsafeComparer.INSTANCE.compareTo(byteArr1, byteArr2);
if (difference != 0) {
return difference;
}
- sizeA += aShort1;
- sizeB += aShort2;
} else {
int dimFieldA = CarbonUnsafe.unsafe.getInt(baseObjectL, rowA + sizeA);
sizeA += 4;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/bfa306b8/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeInmemoryMergeHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeInmemoryMergeHolder.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeInmemoryMergeHolder.java
index ecd97ca..390dbf5 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeInmemoryMergeHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeInmemoryMergeHolder.java
@@ -43,6 +43,8 @@ public class UnsafeInmemoryMergeHolder implements Comparable<UnsafeInmemoryMerge
private Object baseObject;
+ private byte index;
+
public UnsafeInmemoryMergeHolder(UnsafeCarbonRowPage rowPage, byte index) {
this.actualSize = rowPage.getBuffer().getActualSize();
this.rowPage = rowPage;
@@ -50,7 +52,7 @@ public class UnsafeInmemoryMergeHolder implements Comparable<UnsafeInmemoryMerge
this.comparator = new UnsafeRowComparator(rowPage);
this.baseObject = rowPage.getDataBlock().getBaseObject();
currentRow = new UnsafeCarbonRowForMerge();
- currentRow.index = index;
+ this.index = index;
}
public boolean hasNext() {
@@ -62,7 +64,9 @@ public class UnsafeInmemoryMergeHolder implements Comparable<UnsafeInmemoryMerge
public void readRow() {
address = rowPage.getBuffer().get(counter);
+ currentRow = new UnsafeCarbonRowForMerge();
currentRow.address = address + rowPage.getDataBlock().getBaseOffset();
+ currentRow.index = index;
counter++;
}
[2/2] incubator-carbondata git commit: [CARBONDATA-960] Fix Unsafe
merge sort issue.This closes #819
Posted by gv...@apache.org.
[CARBONDATA-960] Fix Unsafe merge sort issue.This closes #819
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/278ed4d5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/278ed4d5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/278ed4d5
Branch: refs/heads/master
Commit: 278ed4d540108e86947b9d7770f83e99ffb8276a
Parents: ca07119 bfa306b
Author: Venkata Ramana G <ra...@huawei.com>
Authored: Thu Apr 20 18:02:19 2017 +0530
Committer: Venkata Ramana G <ra...@huawei.com>
Committed: Thu Apr 20 18:02:19 2017 +0530
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 2 +-
.../apache/carbondata/core/util/ByteUtil.java | 63 --------------------
.../unsafe/comparator/UnsafeRowComparator.java | 34 ++++++-----
.../holder/UnsafeInmemoryMergeHolder.java | 6 +-
4 files changed, 26 insertions(+), 79 deletions(-)
----------------------------------------------------------------------