You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@drill.apache.org by ja...@apache.org on 2014/05/07 03:20:56 UTC
[2/6] git commit: DRILL-644: Fix char length and position calculation
for UTF-8 strings in StringFunctionUtil
DRILL-644: Fix char length and position calculation for UTF-8 strings in StringFunctionUtil
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/0f65fd8d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/0f65fd8d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/0f65fd8d
Branch: refs/heads/master
Commit: 0f65fd8d3bb40300e5455f31603d41dd9a149bb6
Parents: 3af2344
Author: Aditya Kishore <ad...@maprtech.com>
Authored: Tue May 6 01:25:09 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Tue May 6 15:40:10 2014 -0700
----------------------------------------------------------------------
.../exec/expr/fn/impl/StringFunctionUtil.java | 48 +++++++++++---------
.../exec/physical/impl/TestStringFunctions.java | 37 ++++++---------
.../resources/functions/string/testSubstr.json | 9 ++--
3 files changed, 46 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0f65fd8d/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctionUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctionUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctionUtil.java
index 0096a13..fbdab8e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctionUtil.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctionUtil.java
@@ -17,6 +17,8 @@
*/
package org.apache.drill.exec.expr.fn.impl;
+import org.apache.drill.common.exceptions.DrillRuntimeException;
+
import io.netty.buffer.ByteBuf;
public class StringFunctionUtil {
@@ -26,15 +28,9 @@ public class StringFunctionUtil {
public static int getUTF8CharLength(ByteBuf buffer, int start, int end) {
int charCount = 0;
- for (int id = start; id < end; id++) {
- byte currentByte = buffer.getByte(id);
-
- if (currentByte < 0x128 || // 1-byte char. First byte is 0xxxxxxx.
- (currentByte & 0xE0) == 0xC0 || // 2-byte char. First byte is 110xxxxx
- (currentByte & 0xF0) == 0xE0 || // 3-byte char. First byte is 1110xxxx
- (currentByte & 0xF8) == 0xF0) { //4-byte char. First byte is 11110xxx
- charCount ++; //Advance the counter, since we find one char.
- }
+ for (int idx = start, charLen = 0; idx < end; idx += charLen) {
+ charLen = utf8CharLen(buffer, idx);
+ ++charCount; //Advance the counter, since we find one char.
}
return charCount;
}
@@ -46,21 +42,14 @@ public class StringFunctionUtil {
public static int getUTF8CharPosition(ByteBuf buffer, int start, int end, int charLength) {
int charCount = 0;
- if (start >=end)
+ if (start >= end)
return -1; //wrong input here.
- for (int id = start; id < end; id++) {
-
- byte currentByte = buffer.getByte(id);
-
- if (currentByte < 0x128 || // 1-byte char. First byte is 0xxxxxxx.
- (currentByte & 0xE0) == 0xC0 || // 2-byte char. First byte is 110xxxxx
- (currentByte & 0xF0) == 0xE0 || // 3-byte char. First byte is 1110xxxx
- (currentByte & 0xF8) == 0xF0) { //4-byte char. First byte is 11110xxx
- charCount ++; //Advance the counter, since we find one char.
- if (charCount == charLength + 1) {
- return id;
- }
+ for (int idx = start, charLen = 0; idx < end; idx += charLen) {
+ charLen = utf8CharLen(buffer, idx);
+ ++charCount; //Advance the counter, since we find one char.
+ if (charCount == charLength + 1) {
+ return idx;
}
}
return end;
@@ -154,4 +143,19 @@ public class StringFunctionUtil {
return (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F') || (c >= '0' && c <= '9');
}
+ private static int utf8CharLen(ByteBuf buffer, int idx) {
+ byte firstByte = buffer.getByte(idx);
+ if (firstByte >= 0) { // 1-byte char. First byte is 0xxxxxxx.
+ return 1;
+ } else if ((firstByte & 0xE0) == 0xC0) { // 2-byte char. First byte is 110xxxxx
+ return 2;
+ } else if ((firstByte & 0xF0) == 0xE0) { // 3-byte char. First byte is 1110xxxx
+ return 3;
+ } else if ((firstByte & 0xF8) == 0xF0) { //4-byte char. First byte is 11110xxx
+ return 4;
+ }
+ throw new DrillRuntimeException("Unexpected byte 0x" + Integer.toString((int)firstByte & 0xff, 16)
+ + " at position " + idx + " encountered while decoding UTF8 string.");
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0f65fd8d/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestStringFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestStringFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestStringFunctions.java
index 09d1361..cd310b2 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestStringFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestStringFunctions.java
@@ -17,45 +17,36 @@
*/
package org.apache.drill.exec.physical.impl;
-import com.google.common.base.Charsets;
-import com.google.common.io.Resources;
-import com.codahale.metrics.MetricRegistry;
-
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import mockit.Injectable;
import mockit.NonStrictExpectations;
import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.expression.ExpressionPosition;
-import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.util.TestTools;
import org.apache.drill.exec.ExecTest;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
-import org.apache.drill.exec.expr.holders.BigIntHolder;
-import org.apache.drill.exec.expr.holders.VarCharHolder;
-import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.memory.TopLevelAllocator;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.PhysicalPlan;
import org.apache.drill.exec.physical.base.FragmentRoot;
import org.apache.drill.exec.planner.PhysicalPlanReader;
-import org.apache.drill.exec.proto.CoordinationProtos;
-import org.apache.drill.exec.proto.ExecProtos;
import org.apache.drill.exec.proto.BitControl.PlanFragment;
+import org.apache.drill.exec.proto.CoordinationProtos;
import org.apache.drill.exec.rpc.user.UserServer;
import org.apache.drill.exec.server.DrillbitContext;
-import org.apache.drill.exec.vector.BigIntVector;
import org.apache.drill.exec.vector.ValueVector;
-import org.apache.drill.exec.vector.VarBinaryVector;
import org.apache.drill.exec.vector.VarCharVector;
-import org.junit.AfterClass;
-import org.junit.Ignore;
+import org.junit.Rule;
import org.junit.Test;
-import org.junit.Assert;
+import org.junit.rules.TestRule;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
+import com.codahale.metrics.MetricRegistry;
+import com.google.common.base.Charsets;
+import com.google.common.io.Resources;
public class TestStringFunctions extends ExecTest {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestStringFunctions.class);
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestStringFunctions.class);
DrillConfig c = DrillConfig.create();
PhysicalPlanReader reader;
@@ -72,7 +63,7 @@ public class TestStringFunctions extends ExecTest {
int i = 0;
for (ValueVector v : exec) {
if (v instanceof VarCharVector) {
- res[i++] = new String( ((VarCharVector) v).getAccessor().get(0));
+ res[i++] = new String( ((VarCharVector) v).getAccessor().get(0), Charsets.UTF_8);
} else
res[i++] = v.getAccessor().getObject(0);
}
@@ -98,10 +89,10 @@ public class TestStringFunctions extends ExecTest {
while(exec.next()){
Object [] res = getRunResult(exec);
- assertEquals("return count does not match", res.length, expectedResults.length);
+ assertEquals("return count does not match", expectedResults.length, res.length);
for (int i = 0; i<res.length; i++) {
- assertEquals(String.format("column %s does not match", i), res[i], expectedResults[i]);
+ assertEquals(String.format("column %s does not match", i), expectedResults[i], res[i]);
}
}
@@ -199,7 +190,7 @@ public class TestStringFunctions extends ExecTest {
@Test
public void testSubstr(@Injectable final DrillbitContext bitContext,
@Injectable UserServer.UserClientConnection connection) throws Throwable{
- Object [] expected = new Object[] {"abc", "bcd", "bcdef", "bcdef", "", "", "", ""};
+ Object [] expected = new Object[] {"abc", "bcd", "bcdef", "bcdef", "", "", "", "", "भारत", "वर्ष", "वर्ष"};
runTest(bitContext, connection, expected, "functions/string/testSubstr.json");
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0f65fd8d/exec/java-exec/src/test/resources/functions/string/testSubstr.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/functions/string/testSubstr.json b/exec/java-exec/src/test/resources/functions/string/testSubstr.json
index e885381..94467ae 100644
--- a/exec/java-exec/src/test/resources/functions/string/testSubstr.json
+++ b/exec/java-exec/src/test/resources/functions/string/testSubstr.json
@@ -23,15 +23,18 @@
child: 1,
pop:"project",
exprs: [
- { ref: "col1", expr: "substring('abcdef', 1, 3)"},
+ { ref: "col1", expr: "substring('abcdef', 1, 3)"},
{ ref: "col2", expr: "substring('abcdef', 2, 3)"},
{ ref: "col3", expr: "substring('abcdef', 2, 5)"},
{ ref: "col4", expr: "substring('abcdef', 2, 10)"},
{ ref: "col5", expr: "substring('abcdef', 0, 3)"},
{ ref: "col6", expr: "substring('abcdef', -1, 3)"},
{ ref: "col7", expr: "substring('', 1, 2)"},
- { ref: "col8", expr: "substring('abcdef', 10, 2)"}
- ]
+ { ref: "col8", expr: "substring('abcdef', 10, 2)"},
+ { ref: "col9", expr: "substring('भारतवर्ष', 1, 4)"},
+ { ref: "col10", expr: "substring('भारतवर्ष', 5, 4)"},
+ { ref: "col11", expr: "substring('भारतवर्ष', 5, 5)"}
+ ]
},
{
@id: 3,