You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by jh...@apache.org on 2013/09/23 13:51:49 UTC
git commit: TAJO-198: The split function does not work properly.
(jinho)
Updated Branches:
refs/heads/master 7a2061bb9 -> 44bb57dc4
TAJO-198: The split function does not work properly. (jinho)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tajo/commit/44bb57dc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tajo/tree/44bb57dc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tajo/diff/44bb57dc
Branch: refs/heads/master
Commit: 44bb57dc4913c4947dfd6c96c9b939924a29b051
Parents: 7a2061b
Author: jinossy <ji...@gmail.com>
Authored: Mon Sep 23 20:51:16 2013 +0900
Committer: jinossy <ji...@gmail.com>
Committed: Mon Sep 23 20:51:16 2013 +0900
----------------------------------------------------------------------
CHANGES.txt | 2 ++
.../java/org/apache/tajo/datum/NullDatum.java | 10 +++++-
.../main/java/org/apache/tajo/util/Bytes.java | 4 +--
.../tajo/engine/function/string/SplitPart.java | 7 ++--
.../engine/function/TestBuiltinFunctions.java | 34 +++++++++++++++-----
.../java/org/apache/tajo/storage/LazyTuple.java | 24 +++++++-------
.../org/apache/tajo/storage/TestLazyTuple.java | 12 ++-----
.../org/apache/tajo/storage/TestStorages.java | 3 +-
8 files changed, 60 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/44bb57dc/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index dea65e4..0d9fc2b 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -147,6 +147,8 @@ Release 0.2.0 - unreleased
BUG FIXES
+ TAJO-198: The split function does not work properly. (jinho)
+
TAJO-187: The exception 'Some targets cannot be evaluated' is caused in
nested joins. (hyunsik)
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/44bb57dc/tajo-common/src/main/java/org/apache/tajo/datum/NullDatum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/NullDatum.java b/tajo-common/src/main/java/org/apache/tajo/datum/NullDatum.java
index 71d08d3..b5d3ef0 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/NullDatum.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/NullDatum.java
@@ -106,7 +106,7 @@ public class NullDatum extends Datum {
@Override
public int hashCode() {
- return 23244; // one of the prime number
+ return 0; // one of the prime number
}
public static boolean isNull(String val){
@@ -116,4 +116,12 @@ public class NullDatum extends Datum {
public static boolean isNull(byte[] val){
return val == null || val.length == 0 || ((val.length == NULL_CHAR.length) && Bytes.equals(val, NULL_CHAR));
}
+
+ public static boolean isNotNull(String val){
+ return !isNull(val);
+ }
+
+ public static boolean isNotNull(byte[] val){
+ return !isNull(val);
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/44bb57dc/tajo-common/src/main/java/org/apache/tajo/util/Bytes.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/util/Bytes.java b/tajo-common/src/main/java/org/apache/tajo/util/Bytes.java
index 7d5bbd3..cd1ed80 100644
--- a/tajo-common/src/main/java/org/apache/tajo/util/Bytes.java
+++ b/tajo-common/src/main/java/org/apache/tajo/util/Bytes.java
@@ -1396,7 +1396,7 @@ public class Bytes {
*
* @param str the String to parse, may be <code>null</code>
* @param length amount of bytes to str
- * @param separatorChar the separate character
+ * @param separatorChar the ascii separate character
* @param preserveAllTokens if <code>true</code>, adjacent separators are
* @param target the projection target
* treated as empty token separators; if <code>false</code>, adjacent
@@ -1415,7 +1415,7 @@ public class Bytes {
}
if (len == 0) {
- return new byte[0][];
+ return new byte[1][0];
}
List list = new ArrayList();
int i = 0, start = 0;
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/44bb57dc/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/string/SplitPart.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/string/SplitPart.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/string/SplitPart.java
index bbaa441..7a37c33 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/string/SplitPart.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/string/SplitPart.java
@@ -44,8 +44,11 @@ public class SplitPart extends GeneralFunction<TextDatum> {
@Override
public Datum eval(Tuple params) {
- String [] split = StringUtils.split(params.get(0).asChars(), params.get(1).asChars());
- int idx = params.get(2).asInt4();
+ Datum datum = params.get(0);
+ if(datum instanceof NullDatum) return NullDatum.get();
+
+ String [] split = StringUtils.splitByWholeSeparatorPreserveAllTokens(datum.asChars(), params.get(1).asChars(), -1);
+ int idx = params.get(2).asInt4() - 1;
if (split.length > idx) {
return DatumFactory.createText(split[idx]);
} else {
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/44bb57dc/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/function/TestBuiltinFunctions.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/function/TestBuiltinFunctions.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/function/TestBuiltinFunctions.java
index 0384d51..1b354ba 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/function/TestBuiltinFunctions.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/function/TestBuiltinFunctions.java
@@ -19,21 +19,18 @@
package org.apache.tajo.engine.function;
import com.google.common.collect.Maps;
-import org.apache.tajo.client.ResultSetUtil;
+import org.apache.tajo.IntegrationTest;
+import org.apache.tajo.TpchTestBase;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
-import org.apache.tajo.IntegrationTest;
-import org.apache.tajo.TpchTestBase;
import java.io.IOException;
import java.sql.ResultSet;
import java.util.Map;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.*;
@Category(IntegrationTest.class)
public class TestBuiltinFunctions {
@@ -137,7 +134,7 @@ public class TestBuiltinFunctions {
@Test
public void testSplitPart() throws Exception {
- ResultSet res = tpch.execute("select split_part(l_shipinstruct, ' ', 0) from lineitem");
+ ResultSet res = tpch.execute("select split_part(l_shipinstruct, ' ', 1) from lineitem");
String [] result ={
"DELIVER",
@@ -157,8 +154,29 @@ public class TestBuiltinFunctions {
}
@Test
+ public void testSplitPartByString() throws Exception {
+ ResultSet res = tpch.execute("select split_part(l_shipinstruct, 'KE', 1) from lineitem");
+
+ String [] result ={
+ "DELIVER IN PERSON",
+ "TA",
+ "TA",
+ "NONE",
+ "TA"
+ };
+
+ for (int i = 0; i < result.length; i++) {
+ assertTrue(res.next());
+ assertEquals(result[i], res.getString(1));
+ }
+ assertFalse(res.next());
+
+ res.close();
+ }
+
+ @Test
public void testSplitPartNested() throws Exception {
- ResultSet res = tpch.execute("select split_part(split_part(l_shipinstruct, ' ', 0), 'A', 1) from lineitem");
+ ResultSet res = tpch.execute("select split_part(split_part(l_shipinstruct, ' ', 1), 'A', 2) from lineitem");
String [] result ={
"",
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/44bb57dc/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/LazyTuple.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/LazyTuple.java b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/LazyTuple.java
index c364ae2..0e50bed 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/LazyTuple.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/LazyTuple.java
@@ -108,13 +108,15 @@ public class LazyTuple implements Tuple {
//////////////////////////////////////////////////////
@Override
public Datum get(int fieldId) {
- if(values[fieldId] != null)
+ if (values[fieldId] != null)
return values[fieldId];
- else if (textBytes.length > fieldId && (textBytes[fieldId] != null)) {
+ else if (textBytes.length <= fieldId) {
+ values[fieldId] = NullDatum.get(); // split error. (col : 3, separator: ',', row text: "a,")
+ } else if (textBytes[fieldId] != null) {
values[fieldId] = createByTextBytes(schema.getColumn(fieldId).getDataType().getType(), textBytes[fieldId]);
textBytes[fieldId] = null;
} else {
- values[fieldId] = NullDatum.get();
+ //non-projection
}
return values[fieldId];
}
@@ -277,17 +279,17 @@ public class LazyTuple implements Tuple {
private static Datum createByTextBytes(TajoDataTypes.Type type, byte[] val) {
switch (type) {
case BOOLEAN:
- return NullDatum.isNull(val) ? NullDatum.get() : DatumFactory.createBool(new String(val));
+ return NullDatum.isNotNull(val) ? DatumFactory.createBool(new String(val)) : NullDatum.get();
case INT2:
- return NullDatum.isNull(val) ? NullDatum.get() : DatumFactory.createInt2(new String(val));
+ return NullDatum.isNotNull(val) ? DatumFactory.createInt2(new String(val)) : NullDatum.get();
case INT4:
- return NullDatum.isNull(val) ? NullDatum.get() : DatumFactory.createInt4(new String(val));
+ return NullDatum.isNotNull(val) ? DatumFactory.createInt4(new String(val)) : NullDatum.get();
case INT8:
- return NullDatum.isNull(val) ? NullDatum.get() : DatumFactory.createInt8(new String(val));
+ return NullDatum.isNotNull(val) ? DatumFactory.createInt8(new String(val)) : NullDatum.get();
case FLOAT4:
- return NullDatum.isNull(val) ? NullDatum.get() : DatumFactory.createFloat4(new String(val));
+ return NullDatum.isNotNull(val) ? DatumFactory.createFloat4(new String(val)) : NullDatum.get();
case FLOAT8:
- return NullDatum.isNull(val) ? NullDatum.get() : DatumFactory.createFloat8(new String(val));
+ return NullDatum.isNotNull(val) ? DatumFactory.createFloat8(new String(val)) : NullDatum.get();
case CHAR:
return DatumFactory.createChar(new String(val).trim());
case TEXT:
@@ -297,9 +299,9 @@ public class LazyTuple implements Tuple {
case BLOB:
return DatumFactory.createBlob(Base64.decodeBase64(val));
case INET4:
- return NullDatum.isNull(val) ? NullDatum.get() : DatumFactory.createInet4(new String(val));
+ return NullDatum.isNotNull(val) ? DatumFactory.createInet4(new String(val)) : NullDatum.get();
case ARRAY:
- return NullDatum.isNull(val) ? NullDatum.get() : StorageGsonHelper.getInstance().fromJson(new String(val), Datum.class);
+ return NullDatum.isNotNull(val) ? StorageGsonHelper.getInstance().fromJson(new String(val), Datum.class) : NullDatum.get();
case NULL:
return NullDatum.get();
default:
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/44bb57dc/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/TestLazyTuple.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/TestLazyTuple.java b/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/TestLazyTuple.java
index 9dd17e2..3ca0789 100644
--- a/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/TestLazyTuple.java
+++ b/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/TestLazyTuple.java
@@ -124,14 +124,6 @@ public class TestLazyTuple {
assertTrue(t1.getFloat(11).asFloat4() == 0.76f);
}
- public VTuple createVTuple(int size) {
- VTuple vtuple = new VTuple(size);
- for (int i = 0; i < size; i++) {
- vtuple.put(i, NullDatum.get());
- }
- return vtuple;
- }
-
@Test
public void testEquals() {
int colNum = schema.getColumnNum();
@@ -148,7 +140,7 @@ public class TestLazyTuple {
assertEquals(t1, t2);
- Tuple t3 = createVTuple(colNum);
+ Tuple t3 = new VTuple(colNum);
t3.put(0, DatumFactory.createInt4(1));
t3.put(1, DatumFactory.createInt4(2));
t3.put(3, DatumFactory.createInt4(2));
@@ -177,7 +169,7 @@ public class TestLazyTuple {
assertEquals(t1.hashCode(), t2.hashCode());
- Tuple t3 = createVTuple(colNum);
+ Tuple t3 = new VTuple(colNum);
t3.put(0, DatumFactory.createInt4(1));
t3.put(1, DatumFactory.createInt4(2));
t3.put(3, DatumFactory.createInt4(2));
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/44bb57dc/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/TestStorages.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/TestStorages.java b/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/TestStorages.java
index 7589996..a1e0087 100644
--- a/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/TestStorages.java
+++ b/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/TestStorages.java
@@ -43,7 +43,6 @@ import java.util.Arrays;
import java.util.Collection;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
@RunWith(Parameterized.class)
@@ -171,7 +170,7 @@ public class TestStorages {
Tuple tuple;
while ((tuple = scanner.next()) != null) {
if (storeType == StoreType.RCFILE || storeType == StoreType.TREVNI || storeType == StoreType.CSV) {
- assertTrue(tuple.get(0) == null || tuple.get(0) instanceof NullDatum);
+ assertTrue(tuple.get(0) == null);
}
assertEquals(DatumFactory.createInt8(tupleCnt + 2), tuple.getLong(1));
assertEquals(DatumFactory.createFloat4(tupleCnt + 3), tuple.getFloat(2));