You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by jh...@apache.org on 2016/02/15 09:29:07 UTC
tajo git commit: TAJO-2074: Upgrade hadoop and netty.
Repository: tajo
Updated Branches:
refs/heads/master d73f129b4 -> df2b08033
TAJO-2074: Upgrade hadoop and netty.
Closes #961
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/df2b0803
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/df2b0803
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/df2b0803
Branch: refs/heads/master
Commit: df2b08033f1d87ac6cee85f8d4564d57533403ae
Parents: d73f129
Author: Jinho Kim <jh...@apache.org>
Authored: Mon Feb 15 17:28:06 2016 +0900
Committer: Jinho Kim <jh...@apache.org>
Committed: Mon Feb 15 17:28:06 2016 +0900
----------------------------------------------------------------------
CHANGES | 2 +
.../org/apache/tajo/TajoTestingCluster.java | 1 -
.../java/org/apache/tajo/util/BytesUtils.java | 56 ++++++++++++++++++++
.../engine/planner/UniformRangePartition.java | 10 +++-
tajo-project/pom.xml | 7 ++-
.../tajo/storage/TestByteBufLineReader.java | 2 +-
.../apache/tajo/storage/TestFileTablespace.java | 8 +--
.../tajo/storage/raw/TestDirectRawFile.java | 2 +-
8 files changed, 75 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/df2b0803/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 8a52067..ad580aa 100644
--- a/CHANGES
+++ b/CHANGES
@@ -192,6 +192,8 @@ Release 0.12.0 - unreleased
TASKS
+ TAJO-2074: Upgrade hadoop and netty. (jinho)
+
TAJO-1939: Implement PgSQLTablespace::getTableVolume() method. (jihoon)
TAJO-2017: Replace manual array copy with Collection.
http://git-wip-us.apache.org/repos/asf/tajo/blob/df2b0803/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java
----------------------------------------------------------------------
diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java
index 3863203..4e7d236 100644
--- a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java
+++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java
@@ -263,7 +263,6 @@ public class TajoTestingCluster {
builder.hosts(hosts);
builder.numDataNodes(servers);
builder.format(true);
- builder.storagesPerDatanode(1);
builder.waitSafeMode(true);
this.dfsCluster = builder.build();
http://git-wip-us.apache.org/repos/asf/tajo/blob/df2b0803/tajo-common/src/main/java/org/apache/tajo/util/BytesUtils.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/util/BytesUtils.java b/tajo-common/src/main/java/org/apache/tajo/util/BytesUtils.java
index 11d77ce..eb228a8 100644
--- a/tajo-common/src/main/java/org/apache/tajo/util/BytesUtils.java
+++ b/tajo-common/src/main/java/org/apache/tajo/util/BytesUtils.java
@@ -18,11 +18,15 @@
package org.apache.tajo.util;
+import io.netty.buffer.ByteBuf;
+
import java.io.ByteArrayOutputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import static io.netty.util.internal.StringUtil.isSurrogate;
+
/**
* Extra utilities for bytes
*/
@@ -250,4 +254,56 @@ public class BytesUtils {
public static byte [] trimBytes(byte [] bytes) {
return new String(bytes).trim().getBytes();
}
+
+ /**
+ * this is an implementation copied from ByteBufUtil in netty4
+ */
+ public static int writeUtf8(ByteBuf buffer, char[] chars, boolean ignoreSurrogate) {
+ int oldWriterIndex = buffer.writerIndex();
+ int writerIndex = oldWriterIndex;
+
+ // We can use the _set methods as these not need to do any index checks and reference checks.
+ // This is possible as we called ensureWritable(...) before.
+ for (int i = 0; i < chars.length; i++) {
+ char c = chars[i];
+ if (c < 0x80) {
+ buffer.setByte(writerIndex++, (byte) c);
+ } else if (c < 0x800) {
+ buffer.setByte(writerIndex++, (byte) (0xc0 | (c >> 6)));
+ buffer.setByte(writerIndex++, (byte) (0x80 | (c & 0x3f)));
+ } else if (!ignoreSurrogate && isSurrogate(c)) {
+ if (!Character.isHighSurrogate(c)) {
+ throw new IllegalArgumentException("Invalid encoding. " +
+ "Expected high (leading) surrogate at index " + i + " but got " + c);
+ }
+ final char c2;
+ try {
+ // Surrogate Pair consumes 2 characters. Optimistically try to get the next character to avoid
+ // duplicate bounds checking with charAt. If an IndexOutOfBoundsException is thrown we will
+ // re-throw a more informative exception describing the problem.
+ c2 = chars[++i];
+ } catch (IndexOutOfBoundsException e) {
+ throw new IllegalArgumentException("Underflow. " +
+ "Expected low (trailing) surrogate at index " + i + " but no more characters found.", e);
+ }
+ if (!Character.isLowSurrogate(c2)) {
+ throw new IllegalArgumentException("Invalid encoding. " +
+ "Expected low (trailing) surrogate at index " + i + " but got " + c2);
+ }
+ int codePoint = Character.toCodePoint(c, c2);
+ // See http://www.unicode.org/versions/Unicode7.0.0/ch03.pdf#G2630.
+ buffer.setByte(writerIndex++, (byte) (0xf0 | (codePoint >> 18)));
+ buffer.setByte(writerIndex++, (byte) (0x80 | ((codePoint >> 12) & 0x3f)));
+ buffer.setByte(writerIndex++, (byte) (0x80 | ((codePoint >> 6) & 0x3f)));
+ buffer.setByte(writerIndex++, (byte) (0x80 | (codePoint & 0x3f)));
+ } else {
+ buffer.setByte(writerIndex++, (byte) (0xe0 | (c >> 12)));
+ buffer.setByte(writerIndex++, (byte) (0x80 | ((c >> 6) & 0x3f)));
+ buffer.setByte(writerIndex++, (byte) (0x80 | (c & 0x3f)));
+ }
+ }
+ // update the writerIndex without any extra checks for performance reasons
+ buffer.writerIndex(writerIndex);
+ return writerIndex - oldWriterIndex;
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/df2b0803/tajo-core/src/main/java/org/apache/tajo/engine/planner/UniformRangePartition.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/UniformRangePartition.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/UniformRangePartition.java
index 7c26857..02e397d 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/UniformRangePartition.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/UniformRangePartition.java
@@ -20,7 +20,8 @@ package org.apache.tajo.engine.planner;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
-import com.sun.tools.javac.util.Convert;
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.SortSpec;
import org.apache.tajo.common.TajoDataTypes;
@@ -622,7 +623,12 @@ public class UniformRangePartition extends RangePartitionAlgorithm {
}
}
- end.put(i, DatumFactory.createText(Convert.chars2utf(lastChars)));
+ ByteBuf byteBuf = Unpooled.buffer(lastChars.length * 3);
+ BytesUtils.writeUtf8(byteBuf, lastChars, true);
+ byte[] bytes = new byte[byteBuf.readableBytes()];
+ byteBuf.getBytes(0, bytes);
+
+ end.put(i, DatumFactory.createText(bytes));
}
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/df2b0803/tajo-project/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-project/pom.xml b/tajo-project/pom.xml
index 59d2b46..87fa8aa 100644
--- a/tajo-project/pom.xml
+++ b/tajo-project/pom.xml
@@ -33,11 +33,11 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
- <hadoop.version>2.7.1</hadoop.version>
+ <hadoop.version>2.7.2</hadoop.version>
<protobuf.version>2.5.0</protobuf.version>
<hbase.version>1.1.1</hbase.version>
<hive.version>1.1.0</hive.version>
- <netty.version>4.0.33.Final</netty.version>
+ <netty.version>4.0.34.Final</netty.version>
<jersey.version>2.6</jersey.version>
<jetty.version>6.1.26</jetty.version>
<tajo.root>${project.parent.relativePath}/..</tajo.root>
@@ -401,7 +401,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
- <version>3.3</version>
+ <version>3.5</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
@@ -635,7 +635,6 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
- <version>3.3</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
http://git-wip-us.apache.org/repos/asf/tajo/blob/df2b0803/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestByteBufLineReader.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestByteBufLineReader.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestByteBufLineReader.java
index 2cee3bb..18a9b85 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestByteBufLineReader.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestByteBufLineReader.java
@@ -87,7 +87,7 @@ public class TestByteBufLineReader {
final Configuration conf = TestFileTablespace.getTestHdfsConfiguration();
final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
- .numDataNodes(2).storagesPerDatanode(1).format(true).build();
+ .numDataNodes(2).format(true).build();
TajoConf tajoConf = new TajoConf(conf);
tajoConf.setVar(TajoConf.ConfVars.ROOT_DIR, cluster.getFileSystem().getUri() + "/tajo");
http://git-wip-us.apache.org/repos/asf/tajo/blob/df2b0803/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java
index 0370302..58025c3 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java
@@ -122,7 +122,7 @@ public class TestFileTablespace {
public void testGetSplit() throws Exception {
final Configuration hdfsConf = getTestHdfsConfiguration();
final MiniDFSCluster cluster = new MiniDFSCluster.Builder(hdfsConf)
- .numDataNodes(1).storagesPerDatanode(1).format(true).build();
+ .numDataNodes(1).format(true).build();
int testCount = 10;
Path tablePath = new Path("/testGetSplit");
@@ -174,7 +174,7 @@ public class TestFileTablespace {
public void testZeroLengthSplit() throws Exception {
final Configuration hdfsConf = getTestHdfsConfiguration();
final MiniDFSCluster cluster = new MiniDFSCluster.Builder(hdfsConf)
- .numDataNodes(1).storagesPerDatanode(1).format(true).build();
+ .numDataNodes(1).format(true).build();
int testCount = 10;
Path tablePath = new Path("/testZeroLengthSplit");
@@ -222,7 +222,7 @@ public class TestFileTablespace {
hdfsConf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 2);
hdfsConf.setBoolean(DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED, true);
final MiniDFSCluster cluster = new MiniDFSCluster.Builder(hdfsConf)
- .numDataNodes(2).storagesPerDatanode(1).format(true).build();
+ .numDataNodes(2).format(true).build();
int testCount = 10;
Path tablePath = new Path("/testGetSplitWithBlockStorageLocationsBatching");
@@ -266,7 +266,7 @@ public class TestFileTablespace {
final Configuration hdfsConf = getTestHdfsConfiguration();
final MiniDFSCluster cluster = new MiniDFSCluster.Builder(hdfsConf)
- .numDataNodes(1).storagesPerDatanode(1).format(true).build();
+ .numDataNodes(1).format(true).build();
URI uri = URI.create(cluster.getFileSystem().getUri() + "/tajo");
try {
http://git-wip-us.apache.org/repos/asf/tajo/blob/df2b0803/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java
index d027fb8..fb69fa4 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java
@@ -101,7 +101,7 @@ public class TestDirectRawFile {
cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(1)
.format(true)
- .storagesPerDatanode(1).build();
+ .build();
fs = cluster.getFileSystem();
}