You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@drill.apache.org by ja...@apache.org on 2014/05/07 03:20:57 UTC
[3/6] git commit: DRILL-645: Correct string encodings in code template
DRILL-645: Correct string encodings in code template
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/9b34f23d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/9b34f23d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/9b34f23d
Branch: refs/heads/master
Commit: 9b34f23dd8291c7c4c88e0421d02d56e65ae0cd2
Parents: 0f65fd8
Author: Aditya Kishore <ad...@maprtech.com>
Authored: Tue May 6 02:42:59 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Tue May 6 15:40:38 2014 -0700
----------------------------------------------------------------------
.../codegen/templates/CastFunctionsSrcVarLen.java | 16 ++++++++--------
.../src/main/codegen/templates/CastVarCharDate.java | 2 +-
.../main/codegen/templates/CastVarCharInterval.java | 2 +-
.../DateToCharFunctions.java | 2 +-
.../ToDateTypeFunctions.java | 4 ++--
.../templates/Decimal/CastVarCharDecimal.java | 10 +++++-----
.../codegen/templates/ObjectInspectorHelper.java | 4 +++-
.../src/main/codegen/templates/ValueHolders.java | 10 +++++++++-
.../codegen/templates/VariableLengthVectors.java | 12 ++++++++++--
.../drill/exec/physical/impl/TestHiveUDFs.java | 14 +++++++-------
10 files changed, 47 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9b34f23d/exec/java-exec/src/main/codegen/templates/CastFunctionsSrcVarLen.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/CastFunctionsSrcVarLen.java b/exec/java-exec/src/main/codegen/templates/CastFunctionsSrcVarLen.java
index 5855381..aa216cd 100644
--- a/exec/java-exec/src/main/codegen/templates/CastFunctionsSrcVarLen.java
+++ b/exec/java-exec/src/main/codegen/templates/CastFunctionsSrcVarLen.java
@@ -21,7 +21,7 @@
{
byte[] buf = new byte[in.end - in.start];
in.buffer.getBytes(in.start, buf, 0, in.end - in.start);
- throw new NumberFormatException(new String(buf));
+ throw new NumberFormatException(new String(buf, com.google.common.base.Charsets.UTF_8));
}
</#macro>
@@ -58,7 +58,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc{
in.buffer.getBytes(in.start, buf, 0, in.end - in.start);
//TODO: need capture format exception, and issue SQLERR code.
- out.value = ${type.javaType}.parse${type.parse}(new String(buf));
+ out.value = ${type.javaType}.parse${type.parse}(new String(buf, com.google.common.base.Charsets.UTF_8));
<#elseif type.to=="Int" || type.to == "BigInt">
@@ -66,7 +66,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc{
//empty, not a valid number
byte[] buf = new byte[in.end - in.start];
in.buffer.getBytes(in.start, buf, 0, in.end - in.start);
- throw new NumberFormatException(new String(buf));
+ throw new NumberFormatException(new String(buf, com.google.common.base.Charsets.UTF_8));
}
int readIndex = in.start;
@@ -77,7 +77,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc{
//only one single '-'
byte[] buf = new byte[in.end - in.start];
in.buffer.getBytes(in.start, buf, 0, in.end - in.start);
- throw new NumberFormatException(new String(buf));
+ throw new NumberFormatException(new String(buf, com.google.common.base.Charsets.UTF_8));
}
int radix = 10;
@@ -91,13 +91,13 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc{
if (digit == -1) {
byte[] buf = new byte[in.end - in.start];
in.buffer.getBytes(in.start, buf, 0, in.end - in.start);
- throw new NumberFormatException(new String(buf));
+ throw new NumberFormatException(new String(buf, com.google.common.base.Charsets.UTF_8));
}
//overflow
if (max > result) {
byte[] buf = new byte[in.end - in.start];
in.buffer.getBytes(in.start, buf, 0, in.end - in.start);
- throw new NumberFormatException(new String(buf));
+ throw new NumberFormatException(new String(buf, com.google.common.base.Charsets.UTF_8));
}
${type.primeType} next = result * radix - digit;
@@ -106,7 +106,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc{
if (next > result) {
byte[] buf = new byte[in.end - in.start];
in.buffer.getBytes(in.start, buf, 0, in.end - in.start);
- throw new NumberFormatException(new String(buf));
+ throw new NumberFormatException(new String(buf, com.google.common.base.Charsets.UTF_8));
}
result = next;
}
@@ -116,7 +116,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc{
if (result < 0) {
byte[] buf = new byte[in.end - in.start];
in.buffer.getBytes(in.start, buf, 0, in.end - in.start);
- throw new NumberFormatException(new String(buf));
+ throw new NumberFormatException(new String(buf, com.google.common.base.Charsets.UTF_8));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9b34f23d/exec/java-exec/src/main/codegen/templates/CastVarCharDate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/CastVarCharDate.java b/exec/java-exec/src/main/codegen/templates/CastVarCharDate.java
index 39b8b41..249b555 100644
--- a/exec/java-exec/src/main/codegen/templates/CastVarCharDate.java
+++ b/exec/java-exec/src/main/codegen/templates/CastVarCharDate.java
@@ -54,7 +54,7 @@ public class Cast${type.from}To${type.to} implements DrillSimpleFunc {
byte[] buf = new byte[in.end - in.start];
in.buffer.getBytes(in.start, buf, 0, in.end - in.start);
- String input = new String(buf);
+ String input = new String(buf, com.google.common.base.Charsets.UTF_8);
<#if type.to == "Date">
org.joda.time.format.DateTimeFormatter f = org.apache.drill.exec.expr.fn.impl.DateUtility.getDateTimeFormatter();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9b34f23d/exec/java-exec/src/main/codegen/templates/CastVarCharInterval.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/CastVarCharInterval.java b/exec/java-exec/src/main/codegen/templates/CastVarCharInterval.java
index ebb1bd9..135abdd 100644
--- a/exec/java-exec/src/main/codegen/templates/CastVarCharInterval.java
+++ b/exec/java-exec/src/main/codegen/templates/CastVarCharInterval.java
@@ -55,7 +55,7 @@ public class Cast${type.from}To${type.to} implements DrillSimpleFunc {
byte[] buf = new byte[in.end - in.start];
in.buffer.getBytes(in.start, buf, 0, in.end - in.start);
- String input = new String(buf);
+ String input = new String(buf, com.google.common.base.Charsets.UTF_8);
// Parse the ISO format
org.joda.time.Period period = org.joda.time.Period.parse(input);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9b34f23d/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateToCharFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateToCharFunctions.java b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateToCharFunctions.java
index cd5466c..326b1df 100644
--- a/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateToCharFunctions.java
+++ b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateToCharFunctions.java
@@ -57,7 +57,7 @@ public class G${type}ToChar implements DrillSimpleFunc {
// Get the desired output format and create a DateTimeFormatter
byte[] buf = new byte[right.end - right.start];
right.buffer.getBytes(right.start, buf, 0, right.end - right.start);
- String input = new String(buf);
+ String input = new String(buf, com.google.common.base.Charsets.UTF_8);
format = org.joda.time.format.DateTimeFormat.forPattern(input);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9b34f23d/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/ToDateTypeFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/ToDateTypeFunctions.java b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/ToDateTypeFunctions.java
index 30dfc89..fffaef4 100644
--- a/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/ToDateTypeFunctions.java
+++ b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/ToDateTypeFunctions.java
@@ -50,7 +50,7 @@ public class GTo${type} implements DrillSimpleFunc {
// Get the desired output format
byte[] buf = new byte[right.end - right.start];
right.buffer.getBytes(right.start, buf, 0, right.end - right.start);
- String formatString = new String(buf);
+ String formatString = new String(buf, com.google.common.base.Charsets.UTF_8);
format = org.joda.time.format.DateTimeFormat.forPattern(formatString);
}
@@ -59,7 +59,7 @@ public class GTo${type} implements DrillSimpleFunc {
// Get the input
byte[] buf1 = new byte[left.end - left.start];
left.buffer.getBytes(left.start, buf1, 0, left.end - left.start);
- String input = new String(buf1);
+ String input = new String(buf1, com.google.common.base.Charsets.UTF_8);
<#if type == "Date">
out.value = (org.joda.time.DateMidnight.parse(input, format).withZoneRetainFields(org.joda.time.DateTimeZone.UTC)).getMillis();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9b34f23d/exec/java-exec/src/main/codegen/templates/Decimal/CastVarCharDecimal.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/CastVarCharDecimal.java b/exec/java-exec/src/main/codegen/templates/Decimal/CastVarCharDecimal.java
index b0214f4..e3eb973 100644
--- a/exec/java-exec/src/main/codegen/templates/Decimal/CastVarCharDecimal.java
+++ b/exec/java-exec/src/main/codegen/templates/Decimal/CastVarCharDecimal.java
@@ -107,7 +107,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
// not a valid digit
byte[] buf = new byte[in.end - in.start];
in.buffer.getBytes(in.start, buf, 0, in.end - in.start);
- throw new org.apache.drill.common.exceptions.DrillRuntimeException(new String(buf));
+ throw new org.apache.drill.common.exceptions.DrillRuntimeException(new String(buf, com.google.common.base.Charsets.UTF_8));
}
out.value *= radix;
out.value += next;
@@ -117,7 +117,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
if (((integerEndIndex - integerStartIndex) + out.scale) > out.precision) {
byte[] buf = new byte[in.end - in.start];
in.buffer.getBytes(in.start, buf, 0, in.end - in.start);
- throw new org.apache.drill.common.exceptions.DrillRuntimeException("Precision is insufficient for the provided input: " + new String(buf) + " Precision: " + out.precision +
+ throw new org.apache.drill.common.exceptions.DrillRuntimeException("Precision is insufficient for the provided input: " + new String(buf, com.google.common.base.Charsets.UTF_8) + " Precision: " + out.precision +
" Total Digits: " + (out.scale + (integerEndIndex - integerStartIndex)));
}
@@ -240,7 +240,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
// not a valid digit
byte[] buf = new byte[in.end - in.start];
in.buffer.getBytes(in.start, buf, 0, in.end - in.start);
- throw new NumberFormatException(new String(buf));
+ throw new NumberFormatException(new String(buf, com.google.common.base.Charsets.UTF_8));
}
integerDigits++;
@@ -253,7 +253,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
if (integerDigits + out.scale > out.precision) {
byte[] buf = new byte[in.end - in.start];
in.buffer.getBytes(in.start, buf, 0, in.end - in.start);
- throw new org.apache.drill.common.exceptions.DrillRuntimeException("Precision is insufficient for the provided input: " + new String(buf) + " Precision: " + out.precision + " Total Digits: " + (out.scale + integerDigits));
+ throw new org.apache.drill.common.exceptions.DrillRuntimeException("Precision is insufficient for the provided input: " + new String(buf, com.google.common.base.Charsets.UTF_8) + " Precision: " + out.precision + " Total Digits: " + (out.scale + integerDigits));
}
@@ -313,7 +313,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
// not a valid digit
byte[] buf = new byte[in.end - in.start];
in.buffer.getBytes(in.start, buf, 0, in.end - in.start);
- throw new NumberFormatException(new String(buf));
+ throw new NumberFormatException(new String(buf, com.google.common.base.Charsets.UTF_8));
}
int value = (out.getInteger(decimalBufferIndex) * radix) + next;
out.setInteger(decimalBufferIndex, value);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9b34f23d/exec/java-exec/src/main/codegen/templates/ObjectInspectorHelper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/ObjectInspectorHelper.java b/exec/java-exec/src/main/codegen/templates/ObjectInspectorHelper.java
index 61078c7..3c72a96 100644
--- a/exec/java-exec/src/main/codegen/templates/ObjectInspectorHelper.java
+++ b/exec/java-exec/src/main/codegen/templates/ObjectInspectorHelper.java
@@ -23,8 +23,10 @@
package org.apache.drill.exec.expr.fn.impl.hive;
import com.sun.codemodel.*;
+
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.exec.expr.DirectExpression;
import org.apache.drill.exec.expr.TypeHelper;
import org.apache.drill.exec.expr.holders.*;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -153,7 +155,7 @@ public class ObjectInspectorHelper {
<#elseif entry.hiveType == "STRING">
JVar data = jc._else().decl(m.directClass(byte[].class.getCanonicalName()), "data",
castedOI.invoke("getPrimitiveJavaObject").arg(returnValue)
- .invoke("getBytes"));
+ .invoke("getBytes").arg(DirectExpression.direct("com.google.common.base.Charsets.UTF_16")));
jc._else().add(returnValueHolder.ref("buffer")
.invoke("setBytes").arg(JExpr.lit(0)).arg(data));
jc._else().assign(returnValueHolder.ref("start"), JExpr.lit(0));
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9b34f23d/exec/java-exec/src/main/codegen/templates/ValueHolders.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/ValueHolders.java b/exec/java-exec/src/main/codegen/templates/ValueHolders.java
index e36e655..7272f4f 100644
--- a/exec/java-exec/src/main/codegen/templates/ValueHolders.java
+++ b/exec/java-exec/src/main/codegen/templates/ValueHolders.java
@@ -136,7 +136,15 @@ public final class ${className} implements ValueHolder{
</#if>
byte[] buf = new byte[end-start];
buffer.getBytes(start, buf, 0, end-start);
- return new String(buf);
+
+ <#switch minor.class>
+ <#case "Var16Char">
+ return new String(buf, com.google.common.base.Charsets.UTF_16);
+ <#break>
+ <#case "VarChar">
+ <#default>
+ return new String(buf, com.google.common.base.Charsets.UTF_8);
+ </#switch>
}
</#if>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9b34f23d/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java b/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java
index 3905bce..6b05ec5 100644
--- a/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java
+++ b/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java
@@ -463,11 +463,19 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
@Override
public void generateTestData(int size){
boolean even = true;
+ <#switch minor.class>
+ <#case "Var16Char">
+ java.nio.charset.Charset charset = Charsets.UTF_16;
+ <#break>
+ <#case "VarChar">
+ <#default>
+ java.nio.charset.Charset charset = Charsets.UTF_8;
+ </#switch>
for(int i =0; i < size; i++, even = !even){
if(even){
- set(i, new String("aaaaa").getBytes(Charsets.UTF_8));
+ set(i, new String("aaaaa").getBytes(charset));
}else{
- set(i, new String("bbbbbbbbbb").getBytes(Charsets.UTF_8));
+ set(i, new String("bbbbbbbbbb").getBytes(charset));
}
}
setValueCount(size);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9b34f23d/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestHiveUDFs.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestHiveUDFs.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestHiveUDFs.java
index dff69e1..eabeda2 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestHiveUDFs.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestHiveUDFs.java
@@ -109,28 +109,28 @@ public class TestHiveUDFs extends ExecTest {
for(int i=0; i<exec.getRecordCount(); i++) {
- String in = new String(str1V.getAccessor().get(i));
- String upper = new String(upperStr1V.getAccessor().get(i));
+ String in = new String(str1V.getAccessor().get(i), Charsets.UTF_16);
+ String upper = new String(upperStr1V.getAccessor().get(i), Charsets.UTF_16);
assertTrue(in.toUpperCase().equals(upper));
long unix_timestamp = unix_timestampV.getAccessor().get(i);
- String concat = new String(concatV.getAccessor().get(i));
+ String concat = new String(concatV.getAccessor().get(i), Charsets.UTF_16);
assertTrue(concat.equals(in+"-"+in));
float flt1 = flt1V.getAccessor().get(i);
- String format_number = new String(format_numberV.getAccessor().get(i));
+ String format_number = new String(format_numberV.getAccessor().get(i), Charsets.UTF_16);
String nullableStr1 = null;
/* DRILL-425
if (!nullableStr1V.getAccessor().isNull(i))
- nullableStr1 = new String(nullableStr1V.getAccessor().get(i));*/
+ nullableStr1 = new String(nullableStr1V.getAccessor().get(i), Charsets.UTF_16);*/
String upperNullableStr1 = null;
/* DRILL-425
if (!upperNullableStr1V.getAccessor().isNull(i))
- upperNullableStr1 = new String(upperNullableStr1V.getAccessor().get(i)); */
+ upperNullableStr1 = new String(upperNullableStr1V.getAccessor().get(i), Charsets.UTF_16); */
assertEquals(nullableStr1 != null, upperNullableStr1 != null);
if (nullableStr1 != null)
@@ -184,7 +184,7 @@ public class TestHiveUDFs extends ExecTest {
for(int i=0; i<exec.getRecordCount(); i++) {
- String str1 = new String(str1V.getAccessor().get(i));
+ String str1 = new String(str1V.getAccessor().get(i), Charsets.UTF_16);
int str1Length = str1LengthV.getAccessor().get(i);
assertTrue(str1.length() == str1Length);