You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by hj...@apache.org on 2014/11/25 15:45:47 UTC
[06/20] tajo git commit: TAJO-1177: Reduce the use of Sun proprietary
API. (Jihun Kang via hyunsik)
TAJO-1177: Reduce the use of Sun proprietary API. (Jihun Kang via hyunsik)
Closes #243
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/a0625a62
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/a0625a62
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/a0625a62
Branch: refs/heads/hbase_storage
Commit: a0625a620bba9781c51f2ee0f47ec0aad1c36767
Parents: 6059a3d
Author: Hyunsik Choi <hy...@apache.org>
Authored: Mon Nov 17 03:30:36 2014 -0800
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Mon Nov 17 03:30:36 2014 -0800
----------------------------------------------------------------------
CHANGES | 3 ++
.../java/org/apache/tajo/datum/TextDatum.java | 5 +-
.../java/org/apache/tajo/util/KeyValueSet.java | 6 +--
.../java/org/apache/tajo/util/StringUtils.java | 57 ++++++++++++++++++++
.../java/org/apache/tajo/util/UnsafeUtil.java | 4 +-
.../org/apache/tajo/datum/TestTextDatum.java | 14 +++++
.../org/apache/tajo/util/TestStringUtil.java | 24 +++++++++
.../org/apache/tajo/client/TestTajoClient.java | 9 ++--
.../apache/tajo/tuple/offheap/HeapTuple.java | 7 ++-
.../apache/tajo/tuple/offheap/UnSafeTuple.java | 7 ++-
10 files changed, 118 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/a0625a62/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index e68f4a0..ef1561a 100644
--- a/CHANGES
+++ b/CHANGES
@@ -47,6 +47,9 @@ Release 0.9.1 - unreleased
TAJO-1169: Some older version of OpenJDK 1.6 does not get default
timezone id. (Jihun Kang via hyunsik)
+ TAJO-1177: Reduce the use of Sun proprietary API.
+ (Jihun Kang via hyunsik)
+
BUG FIXES
TAJO-1162: to_char() returns "-00" second. (Hyoungjun Kim)
http://git-wip-us.apache.org/repos/asf/tajo/blob/a0625a62/tajo-common/src/main/java/org/apache/tajo/datum/TextDatum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/TextDatum.java b/tajo-common/src/main/java/org/apache/tajo/datum/TextDatum.java
index 9a9b37e..8d8e3f8 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/TextDatum.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/TextDatum.java
@@ -20,11 +20,12 @@ package org.apache.tajo.datum;
import com.google.common.primitives.UnsignedBytes;
import com.google.gson.annotations.Expose;
-import com.sun.tools.javac.util.Convert;
+
import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.exception.InvalidCastException;
import org.apache.tajo.exception.InvalidOperationException;
import org.apache.tajo.util.MurmurHash;
+import org.apache.tajo.util.StringUtils;
import java.nio.charset.Charset;
import java.util.Comparator;
@@ -97,7 +98,7 @@ public class TextDatum extends Datum {
@Override
public char[] asUnicodeChars() {
- return Convert.utf2chars(this.bytes);
+ return StringUtils.convertBytesToChars(bytes, DEFAULT_CHARSET);
}
@Override
http://git-wip-us.apache.org/repos/asf/tajo/blob/a0625a62/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java b/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java
index c1e7092..8e3eb2a 100644
--- a/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java
+++ b/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java
@@ -24,7 +24,6 @@ import com.google.gson.annotations.Expose;
import org.apache.tajo.common.ProtoObject;
import org.apache.tajo.json.CommonGsonHelper;
import org.apache.tajo.json.GsonObject;
-import sun.misc.FloatingDecimal;
import java.util.HashMap;
import java.util.Map;
@@ -173,11 +172,10 @@ public class KeyValueSet implements ProtoObject<KeyValueSetProto>, Cloneable, Gs
if (containsKey(key)) {
String strVal = get(key, null);
try {
- sun.misc.FloatingDecimal fd = FloatingDecimal.readJavaFormatString(strVal);
- if (Float.MAX_VALUE < fd.doubleValue()) {
+ if (Float.MAX_VALUE < Double.parseDouble(strVal)) {
throw new IllegalStateException("Parsed value is overflow in float type");
}
- return fd.floatValue();
+ return Float.parseFloat(strVal);
} catch (NumberFormatException nfe) {
throw new IllegalArgumentException("No such a config key: " + key);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/a0625a62/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java b/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java
index 96118ac..38c0fd8 100644
--- a/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java
+++ b/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java
@@ -25,8 +25,13 @@ import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.hadoop.util.SignalLogger;
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
import java.nio.charset.Charset;
+import java.nio.charset.CharsetDecoder;
import java.nio.charset.CharsetEncoder;
+import java.nio.charset.CoderResult;
+import java.nio.charset.CodingErrorAction;
import java.util.Arrays;
import java.util.BitSet;
@@ -310,4 +315,56 @@ public class StringUtils {
return padded;
}
+
+ public static char[] convertBytesToChars(byte[] src, Charset charset) {
+ CharsetDecoder decoder = charset.newDecoder();
+ char[] resultArray = new char[(int) (src.length * decoder.maxCharsPerByte())];
+
+ if (src.length != 0) {
+ ByteBuffer byteBuffer = ByteBuffer.wrap(src);
+ CharBuffer charBuffer = CharBuffer.wrap(resultArray);
+
+ decoder.onMalformedInput(CodingErrorAction.REPLACE).onUnmappableCharacter(CodingErrorAction.REPLACE);
+ decoder.reset();
+
+ CoderResult coderResult = decoder.decode(byteBuffer, charBuffer, true);
+ if (coderResult.isUnderflow()) {
+ coderResult = decoder.flush(charBuffer);
+
+ if (coderResult.isUnderflow()) {
+ if (resultArray.length != charBuffer.position()) {
+ resultArray = Arrays.copyOf(resultArray, charBuffer.position());
+ }
+ }
+ }
+ }
+
+ return resultArray;
+ }
+
+ public static byte[] convertCharsToBytes(char[] src, Charset charset) {
+ CharsetEncoder encoder = charset.newEncoder();
+ byte[] resultArray = new byte[(int) (src.length * encoder.maxBytesPerChar())];
+
+ if (src.length != 0) {
+ CharBuffer charBuffer = CharBuffer.wrap(src);
+ ByteBuffer byteBuffer = ByteBuffer.wrap(resultArray);
+
+ encoder.onMalformedInput(CodingErrorAction.REPLACE).onUnmappableCharacter(CodingErrorAction.REPLACE);
+ encoder.reset();
+
+ CoderResult coderResult = encoder.encode(charBuffer, byteBuffer, true);
+ if (coderResult.isUnderflow()) {
+ coderResult = encoder.flush(byteBuffer);
+
+ if (coderResult.isUnderflow()) {
+ if (resultArray.length != byteBuffer.position()) {
+ resultArray = Arrays.copyOf(resultArray, byteBuffer.position());
+ }
+ }
+ }
+ }
+
+ return resultArray;
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/a0625a62/tajo-common/src/main/java/org/apache/tajo/util/UnsafeUtil.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/util/UnsafeUtil.java b/tajo-common/src/main/java/org/apache/tajo/util/UnsafeUtil.java
index ae88f73..9d6b9b3 100644
--- a/tajo-common/src/main/java/org/apache/tajo/util/UnsafeUtil.java
+++ b/tajo-common/src/main/java/org/apache/tajo/util/UnsafeUtil.java
@@ -95,7 +95,7 @@ public class UnsafeUtil {
}
public static long getAddress(ByteBuffer buffer) {
- Preconditions.checkArgument(buffer instanceof DirectBuffer, "ByteBuffer must be DirectBuffer");
+ Preconditions.checkArgument(buffer.isDirect(), "ByteBuffer must be DirectBuffer");
return ((DirectBuffer)buffer).address();
}
@@ -105,7 +105,7 @@ public class UnsafeUtil {
public static void free(ByteBuffer bb) {
Preconditions.checkNotNull(bb);
- Preconditions.checkState(bb instanceof DirectBuffer);
+ Preconditions.checkState(bb.isDirect());
Cleaner cleaner = ((DirectBuffer) bb).cleaner();
if (cleaner != null) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/a0625a62/tajo-common/src/test/java/org/apache/tajo/datum/TestTextDatum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/test/java/org/apache/tajo/datum/TestTextDatum.java b/tajo-common/src/test/java/org/apache/tajo/datum/TestTextDatum.java
index bf48f78..2624d33 100644
--- a/tajo-common/src/test/java/org/apache/tajo/datum/TestTextDatum.java
+++ b/tajo-common/src/test/java/org/apache/tajo/datum/TestTextDatum.java
@@ -96,4 +96,18 @@ public class TestTextDatum {
Whitebox.setInternalState(Charset.class, "defaultCharset", systemCharSet);
assertEquals(systemCharSet, Charset.defaultCharset());
}
+
+ @Test
+ public void testAsUnicodeChars() {
+ byte[] testStringArray = new byte[] {0x74, 0x61, 0x6A, 0x6F};
+ TextDatum test = new TextDatum(testStringArray);
+
+ assertArrayEquals(new char[] {'t', 'a', 'j', 'o'}, test.asUnicodeChars());
+
+ testStringArray = new byte[] {(byte) 0xED, (byte) 0x83, (byte) 0x80, (byte) 0xEC,
+ (byte) 0xA1, (byte) 0xB0};
+ test = new TextDatum(testStringArray);
+
+ assertArrayEquals(new char[] {'\ud0c0', '\uc870'}, test.asUnicodeChars());
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/a0625a62/tajo-common/src/test/java/org/apache/tajo/util/TestStringUtil.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/test/java/org/apache/tajo/util/TestStringUtil.java b/tajo-common/src/test/java/org/apache/tajo/util/TestStringUtil.java
index 6c732c7..5272586 100644
--- a/tajo-common/src/test/java/org/apache/tajo/util/TestStringUtil.java
+++ b/tajo-common/src/test/java/org/apache/tajo/util/TestStringUtil.java
@@ -18,6 +18,8 @@
package org.apache.tajo.util;
+import java.nio.charset.Charset;
+
import org.apache.commons.lang.CharUtils;
import org.apache.commons.lang.StringEscapeUtils;
import org.junit.Test;
@@ -125,4 +127,26 @@ public class TestStringUtil {
assertArrayEquals(textArray[1].getBytes(), bytesArray[1]);
assertNull(bytesArray[2]);
}
+
+ @Test
+ public void testConvertBytesToChars() {
+ byte[] testStringArray = new byte[] { 0x74, 0x61, 0x6A, 0x6F };
+ assertArrayEquals(new char[] { 't', 'a', 'j', 'o' },
+ StringUtils.convertBytesToChars(testStringArray, Charset.forName("UTF-8")));
+
+ testStringArray = new byte[] { (byte) 0xED, (byte) 0x83, (byte) 0x80, (byte) 0xEC, (byte) 0xA1, (byte) 0xB0 };
+ assertArrayEquals(new char[] { '\ud0c0', '\uc870' },
+ StringUtils.convertBytesToChars(testStringArray, Charset.forName("UTF-8")));
+ }
+
+ @Test
+ public void testConvertCharsToBytes() {
+ char[] testStringArray = new char[] { 't', 'a', 'j', 'o' };
+ assertArrayEquals(new byte[] { 0x74, 0x61, 0x6A, 0x6F },
+ StringUtils.convertCharsToBytes(testStringArray, Charset.forName("UTF-8")));
+
+ testStringArray = new char[] { '\ud0c0', '\uc870' };
+ assertArrayEquals(new byte[] { (byte) 0xED, (byte) 0x83, (byte) 0x80, (byte) 0xEC, (byte) 0xA1, (byte) 0xB0 },
+ StringUtils.convertCharsToBytes(testStringArray, Charset.forName("UTF-8")));
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/a0625a62/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java b/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java
index 30c94f3..6e255ff 100644
--- a/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java
+++ b/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java
@@ -22,8 +22,9 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.protobuf.ServiceException;
-import com.sun.org.apache.commons.logging.Log;
-import com.sun.org.apache.commons.logging.LogFactory;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -39,12 +40,9 @@ import org.apache.tajo.ipc.ClientProtos.QueryHistoryProto;
import org.apache.tajo.ipc.ClientProtos.QueryInfoProto;
import org.apache.tajo.ipc.ClientProtos.SubQueryHistoryProto;
import org.apache.tajo.jdbc.TajoResultSet;
-import org.apache.tajo.master.querymaster.QueryInfo;
import org.apache.tajo.storage.StorageConstants;
import org.apache.tajo.storage.StorageUtil;
import org.apache.tajo.util.CommonTestingUtil;
-import org.apache.tajo.util.history.QueryUnitHistory;
-import org.apache.tajo.util.history.SubQueryHistory;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -57,7 +55,6 @@ import java.sql.SQLException;
import java.util.*;
import static org.junit.Assert.*;
-import static org.junit.Assert.assertEquals;
@Category(IntegrationTest.class)
public class TestTajoClient {
http://git-wip-us.apache.org/repos/asf/tajo/blob/a0625a62/tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/HeapTuple.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/HeapTuple.java b/tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/HeapTuple.java
index e38555c..33f9f1c 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/HeapTuple.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/HeapTuple.java
@@ -20,16 +20,19 @@ package org.apache.tajo.tuple.offheap;
import com.google.protobuf.InvalidProtocolBufferException;
import com.google.protobuf.Message;
-import com.sun.tools.javac.util.Convert;
+
import org.apache.tajo.datum.*;
import org.apache.tajo.exception.UnsupportedException;
import org.apache.tajo.storage.Tuple;
import org.apache.tajo.storage.VTuple;
import org.apache.tajo.util.SizeOf;
+import org.apache.tajo.util.StringUtils;
import org.apache.tajo.util.UnsafeUtil;
+
import sun.misc.Unsafe;
import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
import static org.apache.tajo.common.TajoDataTypes.DataType;
@@ -241,7 +244,7 @@ public class HeapTuple implements Tuple {
byte [] bytes = new byte[len];
UNSAFE.copyMemory(data, BASE_OFFSET + pos, bytes, UnsafeUtil.ARRAY_BYTE_BASE_OFFSET, len);
- return Convert.utf2chars(bytes);
+ return StringUtils.convertBytesToChars(bytes, Charset.forName("UTF-8"));
}
@Override
http://git-wip-us.apache.org/repos/asf/tajo/blob/a0625a62/tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/UnSafeTuple.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/UnSafeTuple.java b/tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/UnSafeTuple.java
index 138386f..b742e6d 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/UnSafeTuple.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/UnSafeTuple.java
@@ -21,18 +21,21 @@ package org.apache.tajo.tuple.offheap;
import com.google.common.base.Preconditions;
import com.google.protobuf.InvalidProtocolBufferException;
import com.google.protobuf.Message;
-import com.sun.tools.javac.util.Convert;
+
import org.apache.tajo.datum.*;
import org.apache.tajo.exception.UnsupportedException;
import org.apache.tajo.storage.Tuple;
import org.apache.tajo.storage.VTuple;
import org.apache.tajo.util.SizeOf;
+import org.apache.tajo.util.StringUtils;
import org.apache.tajo.util.UnsafeUtil;
+
import sun.misc.Unsafe;
import sun.nio.ch.DirectBuffer;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
+import java.nio.charset.Charset;
import static org.apache.tajo.common.TajoDataTypes.DataType;
@@ -278,7 +281,7 @@ public abstract class UnSafeTuple implements Tuple {
byte [] bytes = new byte[len];
UNSAFE.copyMemory(null, pos, bytes, UnsafeUtil.ARRAY_BYTE_BASE_OFFSET, len);
- return Convert.utf2chars(bytes);
+ return StringUtils.convertBytesToChars(bytes, Charset.forName("UTF-8"));
}
@Override