You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ro...@apache.org on 2016/05/31 22:59:49 UTC
svn commit: r1746358 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/data/utils/SedesHelper.java
Author: rohini
Date: Tue May 31 22:59:49 2016
New Revision: 1746358
URL: http://svn.apache.org/viewvc?rev=1746358&view=rev
Log:
PIG-4821: Pig chararray field with special UTF-8 chars as part of tuple join key produces wrong results in Tez (rohini)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/data/utils/SedesHelper.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1746358&r1=1746357&r2=1746358&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue May 31 22:59:49 2016
@@ -141,6 +141,8 @@ PIG-4639: Add better parser for Apache H
BUG FIXES
+PIG-4821: Pig chararray field with special UTF-8 chars as part of tuple join key produces wrong results in Tez (rohini)
+
PIG-4734: TOMAP schema inferring breaks some scripts in type checking for bincond (daijy)
PIG-4786: CROSS will not work correctly with Grace Parallelism (daijy)
Modified: pig/trunk/src/org/apache/pig/data/utils/SedesHelper.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/data/utils/SedesHelper.java?rev=1746358&r1=1746357&r2=1746358&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/data/utils/SedesHelper.java (original)
+++ pig/trunk/src/org/apache/pig/data/utils/SedesHelper.java Tue May 31 22:59:49 2016
@@ -61,25 +61,25 @@ public class SedesHelper {
public static void writeChararray(DataOutput out, String s) throws IOException {
// a char can take up to 3 bytes in the modified utf8 encoding
// used by DataOutput.writeUTF, so use UNSIGNED_SHORT_MAX/3
- if (s.length() < BinInterSedes.UNSIGNED_SHORT_MAX / 3) {
+ byte[] utfBytes = s.getBytes(BinInterSedes.UTF8);
+ int length = utfBytes.length;
+ if (length < BinInterSedes.UNSIGNED_SHORT_MAX) {
out.writeByte(BinInterSedes.SMALLCHARARRAY);
- out.writeUTF(s);
+ out.writeShort(length);
} else {
- byte[] utfBytes = s.getBytes(BinInterSedes.UTF8);
- int length = utfBytes.length;
-
out.writeByte(BinInterSedes.CHARARRAY);
out.writeInt(length);
- out.write(utfBytes);
}
+ out.write(utfBytes);
}
public static String readChararray(DataInput in, byte type) throws IOException {
+ int size;
if (type == BinInterSedes.SMALLCHARARRAY) {
- return in.readUTF();
+ size = in.readUnsignedShort();
+ } else {
+ size = in.readInt();
}
-
- int size = in.readInt();
byte[] buf = new byte[size];
in.readFully(buf);
return new String(buf, BinInterSedes.UTF8);