You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2017/09/16 08:18:16 UTC
spark git commit: [SPARK-21967][CORE]
org.apache.spark.unsafe.types.UTF8String#compareTo Should Compare 8 Bytes at
a Time for Better Performance
Repository: spark
Updated Branches:
refs/heads/master 0bad10d3e -> 73d906722
[SPARK-21967][CORE] org.apache.spark.unsafe.types.UTF8String#compareTo Should Compare 8 Bytes at a Time for Better Performance
## What changes were proposed in this pull request?
* Using 64 bit unsigned long comparison instead of unsigned int comparison in `org.apache.spark.unsafe.types.UTF8String#compareTo` for better performance.
* Making `IS_LITTLE_ENDIAN` a constant for correctness reasons (shouldn't use a non-constant in `compareTo` implementations and it def. is a constant per JVM)
## How was this patch tested?
Build passes and the functionality is widely covered by existing tests as far as I can see.
Author: Armin <me...@obrown.io>
Closes #19180 from original-brownbear/SPARK-21967.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/73d90672
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/73d90672
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/73d90672
Branch: refs/heads/master
Commit: 73d9067226671adb6410ccfb4d5ca2f00283c82b
Parents: 0bad10d
Author: Armin <me...@obrown.io>
Authored: Sat Sep 16 09:18:13 2017 +0100
Committer: Sean Owen <so...@cloudera.com>
Committed: Sat Sep 16 09:18:13 2017 +0100
----------------------------------------------------------------------
.../apache/spark/unsafe/types/UTF8String.java | 24 ++++++++++++++++----
1 file changed, 19 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/73d90672/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java
----------------------------------------------------------------------
diff --git a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java
index 43f5767..dd67f15 100644
--- a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java
+++ b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java
@@ -64,7 +64,8 @@ public final class UTF8String implements Comparable<UTF8String>, Externalizable,
5, 5, 5, 5,
6, 6};
- private static boolean isLittleEndian = ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN;
+ private static final boolean IS_LITTLE_ENDIAN =
+ ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN;
private static final UTF8String COMMA_UTF8 = UTF8String.fromString(",");
public static final UTF8String EMPTY_UTF8 = UTF8String.fromString("");
@@ -220,7 +221,7 @@ public final class UTF8String implements Comparable<UTF8String>, Externalizable,
// After getting the data, we use a mask to mask out data that is not part of the string.
long p;
long mask = 0;
- if (isLittleEndian) {
+ if (IS_LITTLE_ENDIAN) {
if (numBytes >= 8) {
p = Platform.getLong(base, offset);
} else if (numBytes > 4) {
@@ -1097,10 +1098,23 @@ public final class UTF8String implements Comparable<UTF8String>, Externalizable,
@Override
public int compareTo(@Nonnull final UTF8String other) {
int len = Math.min(numBytes, other.numBytes);
- // TODO: compare 8 bytes as unsigned long
- for (int i = 0; i < len; i ++) {
+ int wordMax = (len / 8) * 8;
+ long roffset = other.offset;
+ Object rbase = other.base;
+ for (int i = 0; i < wordMax; i += 8) {
+ long left = getLong(base, offset + i);
+ long right = getLong(rbase, roffset + i);
+ if (left != right) {
+ if (IS_LITTLE_ENDIAN) {
+ return Long.compareUnsigned(Long.reverseBytes(left), Long.reverseBytes(right));
+ } else {
+ return Long.compareUnsigned(left, right);
+ }
+ }
+ }
+ for (int i = wordMax; i < len; i++) {
// In UTF-8, the byte should be unsigned, so we should compare them as unsigned int.
- int res = (getByte(i) & 0xFF) - (other.getByte(i) & 0xFF);
+ int res = (getByte(i) & 0xFF) - (Platform.getByte(rbase, roffset + i) & 0xFF);
if (res != 0) {
return res;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org