You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by em...@apache.org on 2020/08/14 04:23:59 UTC
[arrow] branch master updated: ARROW-9681: [Java] Fix test failures
of Arrow Memory - Core on big-endian platform
This is an automated email from the ASF dual-hosted git repository.
emkornfield pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new cf1c749 ARROW-9681: [Java] Fix test failures of Arrow Memory - Core on big-endian platform
cf1c749 is described below
commit cf1c749098d49a1f1f2f4c7dc1ea645f3102824c
Author: Kazuaki Ishizaki <is...@jp.ibm.com>
AuthorDate: Thu Aug 13 21:23:29 2020 -0700
ARROW-9681: [Java] Fix test failures of Arrow Memory - Core on big-endian platform
This PR fixes two test failures on BE platform by taking care of big-endian in `setBytes()` and `memcpy()`.
```
% mvn -B -Drat.skip=true -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -Dflatc.download.skip=true -rf :arrow-memory-core test
...
[ERROR] TestArrowBuf.testSetBytesBigEndian:145 arrays first differed at element [0]; expected:<0> but was:<7>
[ERROR] TestByteFunctionHelpers.testStringCompare Multiple Failures (2 failures)
expected:<-1> but was:<1>
...
[ERROR] Tests run: 31, Failures: 2, Errors: 0, Skipped: 0
```
Closes #7923 from kiszk/ARROW-9681
Authored-by: Kazuaki Ishizaki <is...@jp.ibm.com>
Signed-off-by: Micah Kornfield <em...@gmail.com>
---
.../java/org/apache/arrow/memory/ArrowBuf.java | 4 +--
.../arrow/memory/util/ByteFunctionHelpers.java | 34 ++++++++++++++++++----
2 files changed, 31 insertions(+), 7 deletions(-)
diff --git a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/ArrowBuf.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/ArrowBuf.java
index e81a92f..ea5e29f 100644
--- a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/ArrowBuf.java
+++ b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/ArrowBuf.java
@@ -163,7 +163,7 @@ public final class ArrowBuf implements AutoCloseable {
* Returns the byte order of elements in this buffer.
*/
public ByteOrder order() {
- return ByteOrder.LITTLE_ENDIAN;
+ return ByteOrder.nativeOrder();
}
/**
@@ -846,7 +846,7 @@ public final class ArrowBuf implements AutoCloseable {
src.position(src.position() + length);
} else {
final ByteOrder originalByteOrder = src.order();
- src.order(ByteOrder.LITTLE_ENDIAN);
+ src.order(order());
try {
// copy word at a time
while (length - 128 >= LONG_SIZE) {
diff --git a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/ByteFunctionHelpers.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/ByteFunctionHelpers.java
index df57128..9579245 100644
--- a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/ByteFunctionHelpers.java
+++ b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/ByteFunctionHelpers.java
@@ -17,6 +17,8 @@
package org.apache.arrow.memory.util;
+import java.nio.ByteOrder;
+
import org.apache.arrow.memory.ArrowBuf;
import org.apache.arrow.memory.BoundsChecking;
import org.apache.arrow.memory.util.hash.ArrowBufHasher;
@@ -28,6 +30,8 @@ import org.apache.arrow.memory.util.hash.SimpleHasher;
public class ByteFunctionHelpers {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ByteFunctionHelpers.class);
+ private static final boolean LITTLE_ENDIAN = ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN;
+
private ByteFunctionHelpers() {}
/**
@@ -152,7 +156,11 @@ public class ByteFunctionHelpers {
long leftLong = MemoryUtil.UNSAFE.getLong(lPos);
long rightLong = MemoryUtil.UNSAFE.getLong(rPos);
if (leftLong != rightLong) {
- return unsignedLongCompare(Long.reverseBytes(leftLong), Long.reverseBytes(rightLong));
+ if (LITTLE_ENDIAN) {
+ return unsignedLongCompare(Long.reverseBytes(leftLong), Long.reverseBytes(rightLong));
+ } else {
+ return unsignedLongCompare(leftLong, rightLong);
+ }
}
lPos += 8;
rPos += 8;
@@ -164,7 +172,11 @@ public class ByteFunctionHelpers {
long leftLong = MemoryUtil.UNSAFE.getLong(lPos);
long rightLong = MemoryUtil.UNSAFE.getLong(rPos);
if (leftLong != rightLong) {
- return unsignedLongCompare(Long.reverseBytes(leftLong), Long.reverseBytes(rightLong));
+ if (LITTLE_ENDIAN) {
+ return unsignedLongCompare(Long.reverseBytes(leftLong), Long.reverseBytes(rightLong));
+ } else {
+ return unsignedLongCompare(leftLong, rightLong);
+ }
}
lPos += 8;
rPos += 8;
@@ -175,7 +187,11 @@ public class ByteFunctionHelpers {
int leftInt = MemoryUtil.UNSAFE.getInt(lPos);
int rightInt = MemoryUtil.UNSAFE.getInt(rPos);
if (leftInt != rightInt) {
- return unsignedIntCompare(Integer.reverseBytes(leftInt), Integer.reverseBytes(rightInt));
+ if (LITTLE_ENDIAN) {
+ return unsignedIntCompare(Integer.reverseBytes(leftInt), Integer.reverseBytes(rightInt));
+ } else {
+ return unsignedIntCompare(leftInt, rightInt);
+ }
}
lPos += 4;
rPos += 4;
@@ -259,7 +275,11 @@ public class ByteFunctionHelpers {
long leftLong = MemoryUtil.UNSAFE.getLong(lPos);
long rightLong = MemoryUtil.UNSAFE.getLong(right, MemoryUtil.BYTE_ARRAY_BASE_OFFSET + rPos);
if (leftLong != rightLong) {
- return unsignedLongCompare(Long.reverseBytes(leftLong), Long.reverseBytes(rightLong));
+ if (LITTLE_ENDIAN) {
+ return unsignedLongCompare(Long.reverseBytes(leftLong), Long.reverseBytes(rightLong));
+ } else {
+ return unsignedLongCompare(leftLong, rightLong);
+ }
}
lPos += 8;
rPos += 8;
@@ -270,7 +290,11 @@ public class ByteFunctionHelpers {
int leftInt = MemoryUtil.UNSAFE.getInt(lPos);
int rightInt = MemoryUtil.UNSAFE.getInt(right, MemoryUtil.BYTE_ARRAY_BASE_OFFSET + rPos);
if (leftInt != rightInt) {
- return unsignedIntCompare(Integer.reverseBytes(leftInt), Integer.reverseBytes(rightInt));
+ if (LITTLE_ENDIAN) {
+ return unsignedIntCompare(Integer.reverseBytes(leftInt), Integer.reverseBytes(rightInt));
+ } else {
+ return unsignedIntCompare(leftInt, rightInt);
+ }
}
lPos += 4;
rPos += 4;