You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/03/26 16:07:33 UTC
[1/2] incubator-carbondata git commit: Fixed message fails with
outofbound exception in dictionary server
Repository: incubator-carbondata
Updated Branches:
refs/heads/master e0f7aca28 -> f0e67c965
Fixed message fails with outofbound exception in dictionary server
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/6a9c79c2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/6a9c79c2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/6a9c79c2
Branch: refs/heads/master
Commit: 6a9c79c281532c10d8f1bdd00da18c6367f9435b
Parents: e0f7aca
Author: ravipesala <ra...@gmail.com>
Authored: Thu Mar 23 19:37:38 2017 +0530
Committer: jackylk <ja...@huawei.com>
Committed: Sun Mar 26 21:36:50 2017 +0530
----------------------------------------------------------------------
.../dictionary/client/DictionaryClient.java | 5 ++-
.../generator/key/DictionaryMessage.java | 22 +++++++----
.../dictionary/server/DictionaryServer.java | 4 +-
.../dictionary/client/DictionaryClientTest.java | 41 +++++++++++++++++---
4 files changed, 58 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/6a9c79c2/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClient.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClient.java b/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClient.java
index d86be99..39f747e 100644
--- a/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClient.java
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClient.java
@@ -28,7 +28,7 @@ import io.netty.channel.ChannelPipeline;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.SocketChannel;
import io.netty.channel.socket.nio.NioSocketChannel;
-
+import io.netty.handler.codec.LengthFieldBasedFrameDecoder;
/**
* Dictionary client to connect to Dictionary server and generate dictionary values
@@ -56,6 +56,9 @@ public class DictionaryClient {
.handler(new ChannelInitializer<SocketChannel>() {
@Override public void initChannel(SocketChannel ch) throws Exception {
ChannelPipeline pipeline = ch.pipeline();
+ // Based on length provided at header, it collects all packets
+ pipeline
+ .addLast("LengthDecoder", new LengthFieldBasedFrameDecoder(1048576, 0, 2, 0, 2));
pipeline.addLast("DictionaryClientHandler", dictionaryClientHandler);
}
});
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/6a9c79c2/core/src/main/java/org/apache/carbondata/core/dictionary/generator/key/DictionaryMessage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/key/DictionaryMessage.java b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/key/DictionaryMessage.java
index e7b14a2..2ae0521 100644
--- a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/key/DictionaryMessage.java
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/key/DictionaryMessage.java
@@ -73,6 +73,10 @@ public class DictionaryMessage {
}
public void writeData(ByteBuf byteBuf) {
+ int startIndex = byteBuf.writerIndex();
+ // Just reserve the bytes to add length of header at last.
+ byteBuf.writeShort(Short.MAX_VALUE);
+
byte[] tableBytes = tableUniqueName.getBytes();
byteBuf.writeInt(tableBytes.length);
byteBuf.writeBytes(tableBytes);
@@ -92,17 +96,22 @@ public class DictionaryMessage {
byteBuf.writeInt(dataBytes.length);
byteBuf.writeBytes(dataBytes);
}
+ int endIndex = byteBuf.writerIndex();
+ // Add the length of message at the starting.it is required while decoding as in TCP protocol
+ // it not guarantee that we receive all data in one packet, so we need to wait to receive all
+ // packets before proceeding to process the message.Based on the length it waits.
+ byteBuf.setShort(startIndex, endIndex - startIndex - 2);
}
private DictionaryMessageType getKeyType(byte type) {
switch (type) {
- case 1 :
+ case 1:
return DictionaryMessageType.DICT_GENERATION;
- case 2 :
+ case 2:
return DictionaryMessageType.TABLE_INTIALIZATION;
- case 3 :
+ case 3:
return DictionaryMessageType.SIZE;
- case 4 :
+ case 4:
return DictionaryMessageType.WRITE_DICTIONARY;
default:
return DictionaryMessageType.DICT_GENERATION;
@@ -150,8 +159,7 @@ public class DictionaryMessage {
}
@Override public String toString() {
- return "DictionaryKey{ columnName='"
- + columnName + '\'' + ", data='" + data + '\'' + ", dictionaryValue=" + dictionaryValue
- + ", type=" + type + '}';
+ return "DictionaryKey{ columnName='" + columnName + '\'' + ", data='" + data + '\''
+ + ", dictionaryValue=" + dictionaryValue + ", type=" + type + '}';
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/6a9c79c2/core/src/main/java/org/apache/carbondata/core/dictionary/server/DictionaryServer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/server/DictionaryServer.java b/core/src/main/java/org/apache/carbondata/core/dictionary/server/DictionaryServer.java
index 38eda0e..e2eaaa3 100644
--- a/core/src/main/java/org/apache/carbondata/core/dictionary/server/DictionaryServer.java
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/server/DictionaryServer.java
@@ -29,7 +29,7 @@ import io.netty.channel.EventLoopGroup;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.SocketChannel;
import io.netty.channel.socket.nio.NioServerSocketChannel;
-
+import io.netty.handler.codec.LengthFieldBasedFrameDecoder;
/**
* Dictionary Server to generate dictionary keys.
@@ -64,6 +64,8 @@ public class DictionaryServer {
bootstrap.childHandler(new ChannelInitializer<SocketChannel>() {
@Override public void initChannel(SocketChannel ch) throws Exception {
ChannelPipeline pipeline = ch.pipeline();
+ // Based on length provided at header, it collects all packets
+ pipeline.addLast("LengthDecoder", new LengthFieldBasedFrameDecoder(1048576, 0, 2, 0, 2));
pipeline.addLast("DictionaryServerHandler", dictionaryServerHandler);
}
});
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/6a9c79c2/core/src/test/java/org/apache/carbondata/core/dictionary/client/DictionaryClientTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/dictionary/client/DictionaryClientTest.java b/core/src/test/java/org/apache/carbondata/core/dictionary/client/DictionaryClientTest.java
index 5d9326b..c7989e4 100644
--- a/core/src/test/java/org/apache/carbondata/core/dictionary/client/DictionaryClientTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/dictionary/client/DictionaryClientTest.java
@@ -38,8 +38,6 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import static org.junit.Assert.assertTrue;
-
/**
* Test class to test dictionary client functionality.
*/
@@ -111,18 +109,51 @@ public class DictionaryClientTest {
// Test dictionary initialization call
empKey.setType(DictionaryMessageType.TABLE_INTIALIZATION);
client.getDictionary(empKey);
-
+ int count = 2;
// Test dictionary generation
- for (int count = 2; count <= 10000; count++) {
+ for (; count <= 10000; count++) {
empKey.setType(DictionaryMessageType.DICT_GENERATION);
empKey.setData("FirstKey" + count);
DictionaryMessage val = client.getDictionary(empKey);
Assert.assertEquals(count, val.getDictionaryValue());
}
+
+ // Test dictionary generation with big messages
+ for (; count <= 10010; count++) {
+ empKey.setType(DictionaryMessageType.DICT_GENERATION);
+ empKey.setData(
+ "FirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKey"
+ + "FirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKey"
+ + "FirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKey"
+ + "FirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKey"
+ + "FirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKey"
+ + "FirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKey"
+ + "FirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKey"
+ + "FirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKey"
+ + "FirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKey"
+ + "FirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKey"
+ + "FirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKey"
+ + "FirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKey"
+ + "FirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKey"
+ + "FirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKey"
+ + "FirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKey"
+ + "FirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKey"
+ + "FirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKey"
+ + "FirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKey"
+ + "FirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKey"
+ + "FirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKey"
+ + "FirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKey"
+ + "FirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKey"
+ + "FirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKey"
+ + "FirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKeyFirstKey"
+ + count);
+ DictionaryMessage val = client.getDictionary(empKey);
+ Assert.assertEquals(count, val.getDictionaryValue());
+ }
// Test size function
empKey.setType(DictionaryMessageType.SIZE);
DictionaryMessage val = client.getDictionary(empKey);
- Assert.assertEquals(10000, val.getDictionaryValue());
+ Assert.assertEquals(10010, val.getDictionaryValue());
client.shutDown();
[2/2] incubator-carbondata git commit: [CARBONDATA-783] Fixed message
fails with outofbound exception in dictionary server This closes #691
Posted by ja...@apache.org.
[CARBONDATA-783] Fixed message fails with outofbound exception in dictionary server This closes #691
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/f0e67c96
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/f0e67c96
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/f0e67c96
Branch: refs/heads/master
Commit: f0e67c965f20d5224e08abdfdd04aa14df4f226f
Parents: e0f7aca 6a9c79c
Author: jackylk <ja...@huawei.com>
Authored: Sun Mar 26 21:37:23 2017 +0530
Committer: jackylk <ja...@huawei.com>
Committed: Sun Mar 26 21:37:23 2017 +0530
----------------------------------------------------------------------
.../dictionary/client/DictionaryClient.java | 5 ++-
.../generator/key/DictionaryMessage.java | 22 +++++++----
.../dictionary/server/DictionaryServer.java | 4 +-
.../dictionary/client/DictionaryClientTest.java | 41 +++++++++++++++++---
4 files changed, 58 insertions(+), 14 deletions(-)
----------------------------------------------------------------------