You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by jh...@apache.org on 2016/03/18 03:14:43 UTC
tajo git commit: TAJO-2092:
TestStorages.testNullHandlingTypesWithProjection occasionally fail.
Repository: tajo
Updated Branches:
refs/heads/master ae707c82d -> d25bec87a
TAJO-2092: TestStorages.testNullHandlingTypesWithProjection occasionally fail.
Closes #982
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/d25bec87
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/d25bec87
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/d25bec87
Branch: refs/heads/master
Commit: d25bec87ac0fe9c819cbcf79e823222fcdad239d
Parents: ae707c8
Author: Jinho Kim <jh...@apache.org>
Authored: Fri Mar 18 11:14:00 2016 +0900
Committer: Jinho Kim <jh...@apache.org>
Committed: Fri Mar 18 11:14:00 2016 +0900
----------------------------------------------------------------------
CHANGES | 3 ++
.../org/apache/tajo/storage/TestStorages.java | 56 +++++++-------------
.../resources/dataset/testVariousTypes.avsc | 3 +-
3 files changed, 24 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/d25bec87/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index b5ff5f4..7fa4087 100644
--- a/CHANGES
+++ b/CHANGES
@@ -112,6 +112,9 @@ Release 0.12.0 - unreleased
BUG FIXES
+ TAJO-2092: TestStorages.testNullHandlingTypesWithProjection occasionally fail.
+ (jinho)
+
TAJO-1793: result row count unmatched for UNION ALL. (jaehwa)
TAJO-2082: Aggregation on a derived table which includes union can cause
http://git-wip-us.apache.org/repos/asf/tajo/blob/d25bec87/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
index 44d8fdc..552dc2e 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
@@ -93,8 +93,7 @@ public class TestStorages {
" { \"name\": \"col8\", \"type\": [\"null\", \"string\"] },\n" +
" { \"name\": \"col9\", \"type\": [\"null\", \"bytes\"] },\n" +
" { \"name\": \"col10\", \"type\": [\"null\", \"bytes\"] },\n" +
- " { \"name\": \"col11\", \"type\": \"null\" },\n" +
- " { \"name\": \"col12\", \"type\": [\"null\", \"bytes\"] }\n" +
+ " { \"name\": \"col11\", \"type\": [\"null\", \"bytes\"] }\n" +
" ]\n" +
"}\n";
@@ -399,9 +398,8 @@ public class TestStorages {
schema.addColumn("col8", Type.TEXT);
schema.addColumn("col9", Type.BLOB);
schema.addColumn("col10", Type.INET4);
- schema.addColumn("col11", Type.NULL_TYPE);
if (handleProtobuf) {
- schema.addColumn("col12", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()));
+ schema.addColumn("col11", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()));
}
KeyValueSet options = new KeyValueSet();
@@ -420,7 +418,7 @@ public class TestStorages {
QueryId queryid = new QueryId("12345", 5);
ProtobufDatumFactory factory = ProtobufDatumFactory.get(TajoIdProtos.QueryIdProto.class.getName());
- VTuple tuple = new VTuple(11 + (handleProtobuf ? 1 : 0));
+ VTuple tuple = new VTuple(10 + (handleProtobuf ? 1 : 0));
tuple.put(new Datum[] {
DatumFactory.createBool(true),
DatumFactory.createChar("hyunsik"),
@@ -432,11 +430,10 @@ public class TestStorages {
DatumFactory.createText("hyunsik"),
DatumFactory.createBlob("hyunsik".getBytes()),
DatumFactory.createInet4("192.168.0.1"),
- NullDatum.get()
});
if (handleProtobuf) {
- tuple.put(11, factory.createDatum(queryid.getProto()));
+ tuple.put(10, factory.createDatum(queryid.getProto()));
}
appender.addTuple(tuple);
@@ -472,10 +469,9 @@ public class TestStorages {
schema.addColumn("col8", Type.TEXT);
schema.addColumn("col9", Type.BLOB);
schema.addColumn("col10", Type.INET4);
- schema.addColumn("col11", Type.NULL_TYPE);
if (handleProtobuf) {
- schema.addColumn("col12", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()));
+ schema.addColumn("col11", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()));
}
KeyValueSet options = new KeyValueSet();
@@ -496,7 +492,7 @@ public class TestStorages {
QueryId queryid = new QueryId("12345", 5);
ProtobufDatumFactory factory = ProtobufDatumFactory.get(TajoIdProtos.QueryIdProto.class.getName());
- int columnNum = 11 + (handleProtobuf ? 1 : 0);
+ int columnNum = 10 + (handleProtobuf ? 1 : 0);
VTuple seedTuple = new VTuple(columnNum);
seedTuple.put(new Datum[]{
DatumFactory.createBool(true), // 0
@@ -508,12 +504,11 @@ public class TestStorages {
DatumFactory.createFloat8(271.9f), // 7
DatumFactory.createText("hyunsik"), // 8
DatumFactory.createBlob("hyunsik".getBytes()),// 9
- DatumFactory.createInet4("192.168.0.1"), // 10
- NullDatum.get(), // 11
+ DatumFactory.createInet4("192.168.0.1") // 10
});
if (handleProtobuf) {
- seedTuple.put(11, factory.createDatum(queryid.getProto())); // 12
+ seedTuple.put(10, factory.createDatum(queryid.getProto())); // 11
}
// Making tuples with different null column positions
@@ -571,10 +566,9 @@ public class TestStorages {
schema.addColumn("col8", Type.TEXT);
schema.addColumn("col9", Type.BLOB);
schema.addColumn("col10", Type.INET4);
- schema.addColumn("col11", Type.NULL_TYPE);
if (handleProtobuf) {
- schema.addColumn("col12", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()));
+ schema.addColumn("col11", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()));
}
KeyValueSet options = new KeyValueSet();
@@ -595,7 +589,7 @@ public class TestStorages {
QueryId queryid = new QueryId("12345", 5);
ProtobufDatumFactory factory = ProtobufDatumFactory.get(TajoIdProtos.QueryIdProto.class.getName());
- int columnNum = 11 + (handleProtobuf ? 1 : 0);
+ int columnNum = 10 + (handleProtobuf ? 1 : 0);
VTuple seedTuple = new VTuple(columnNum);
seedTuple.put(new Datum[]{
DatumFactory.createBool(true), // 0
@@ -607,12 +601,11 @@ public class TestStorages {
DatumFactory.createFloat8(271.9f), // 7
DatumFactory.createText("hyunsik"), // 8
DatumFactory.createBlob("hyunsik".getBytes()),// 9
- DatumFactory.createInet4("192.168.0.1"), // 10
- NullDatum.get(), // 11
+ DatumFactory.createInet4("192.168.0.1") // 10
});
if (handleProtobuf) {
- seedTuple.put(11, factory.createDatum(queryid.getProto())); // 12
+ seedTuple.put(10, factory.createDatum(queryid.getProto())); // 11
}
// Making tuples with different null column positions
@@ -680,14 +673,13 @@ public class TestStorages {
schema.addColumn("col9", Type.TEXT);
schema.addColumn("col10", Type.BLOB);
schema.addColumn("col11", Type.INET4);
- schema.addColumn("col12", Type.NULL_TYPE);
- schema.addColumn("col13", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()));
+ schema.addColumn("col12", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()));
KeyValueSet options = new KeyValueSet();
TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options);
meta.putProperty(StorageConstants.CSVFILE_SERDE, TextSerializerDeserializer.class.getName());
- Path tablePath = new Path(testDir, "testVariousTypes.data");
+ Path tablePath = new Path(testDir, "testRCFileTextSerializeDeserialize.data");
FileTablespace sm = TablespaceManager.getLocalFs();
Appender appender = sm.getAppender(meta, schema, tablePath);
appender.enableStats();
@@ -708,7 +700,6 @@ public class TestStorages {
DatumFactory.createText("jinho"),
DatumFactory.createBlob("hyunsik babo".getBytes()),
DatumFactory.createInet4("192.168.0.1"),
- NullDatum.get(),
factory.createDatum(queryid.getProto())
});
appender.addTuple(tuple);
@@ -749,14 +740,13 @@ public class TestStorages {
schema.addColumn("col9", Type.TEXT);
schema.addColumn("col10", Type.BLOB);
schema.addColumn("col11", Type.INET4);
- schema.addColumn("col12", Type.NULL_TYPE);
- schema.addColumn("col13", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()));
+ schema.addColumn("col12", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()));
KeyValueSet options = new KeyValueSet();
TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options);
meta.putProperty(StorageConstants.RCFILE_SERDE, BinarySerializerDeserializer.class.getName());
- Path tablePath = new Path(testDir, "testVariousTypes.data");
+ Path tablePath = new Path(testDir, "testRCFileBinarySerializeDeserialize.data");
FileTablespace sm = TablespaceManager.getLocalFs();
Appender appender = sm.getAppender(meta, schema, tablePath);
appender.enableStats();
@@ -777,7 +767,6 @@ public class TestStorages {
DatumFactory.createText("jinho"),
DatumFactory.createBlob("hyunsik babo".getBytes()),
DatumFactory.createInet4("192.168.0.1"),
- NullDatum.get(),
factory.createDatum(queryid.getProto())
});
appender.addTuple(tuple);
@@ -818,14 +807,13 @@ public class TestStorages {
schema.addColumn("col9", Type.TEXT);
schema.addColumn("col10", Type.BLOB);
schema.addColumn("col11", Type.INET4);
- schema.addColumn("col12", Type.NULL_TYPE);
- schema.addColumn("col13", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()));
+ schema.addColumn("col12", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()));
KeyValueSet options = new KeyValueSet();
TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options);
meta.putProperty(StorageConstants.SEQUENCEFILE_SERDE, TextSerializerDeserializer.class.getName());
- Path tablePath = new Path(testDir, "testVariousTypes.data");
+ Path tablePath = new Path(testDir, "testSequenceFileTextSerializeDeserialize.data");
FileTablespace sm = TablespaceManager.getLocalFs();
Appender appender = sm.getAppender(meta, schema, tablePath);
appender.enableStats();
@@ -845,7 +833,6 @@ public class TestStorages {
DatumFactory.createText("jinho"),
DatumFactory.createBlob("hyunsik babo".getBytes()),
DatumFactory.createInet4("192.168.0.1"),
- NullDatum.get(),
ProtobufDatumFactory.createDatum(queryid.getProto())
});
appender.addTuple(tuple);
@@ -890,8 +877,7 @@ public class TestStorages {
schema.addColumn("col9", Type.TEXT);
schema.addColumn("col10", Type.BLOB);
schema.addColumn("col11", Type.INET4);
- schema.addColumn("col12", Type.NULL_TYPE);
- schema.addColumn("col13", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()));
+ schema.addColumn("col12", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()));
KeyValueSet options = new KeyValueSet();
TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options);
@@ -905,8 +891,7 @@ public class TestStorages {
QueryId queryid = new QueryId("12345", 5);
- VTuple tuple = new VTuple(13);
- tuple.put(new Datum[] {
+ VTuple tuple = new VTuple(new Datum[] {
DatumFactory.createBool(true),
DatumFactory.createBit((byte) 0x99),
DatumFactory.createChar("jinho"),
@@ -918,7 +903,6 @@ public class TestStorages {
DatumFactory.createText("jinho"),
DatumFactory.createBlob("hyunsik babo".getBytes()),
DatumFactory.createInet4("192.168.0.1"),
- NullDatum.get(),
ProtobufDatumFactory.createDatum(queryid.getProto())
});
appender.addTuple(tuple);
http://git-wip-us.apache.org/repos/asf/tajo/blob/d25bec87/tajo-storage/tajo-storage-hdfs/src/test/resources/dataset/testVariousTypes.avsc
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/resources/dataset/testVariousTypes.avsc b/tajo-storage/tajo-storage-hdfs/src/test/resources/dataset/testVariousTypes.avsc
index d4250a9..f71f052 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/resources/dataset/testVariousTypes.avsc
+++ b/tajo-storage/tajo-storage-hdfs/src/test/resources/dataset/testVariousTypes.avsc
@@ -13,8 +13,7 @@
{ "name": "col8", "type": "string" },
{ "name": "col9", "type": "bytes" },
{ "name": "col10", "type": "bytes" },
- { "name": "col11", "type": "null" },
- { "name": "col12", "type": "bytes" }
+ { "name": "col11", "type": "bytes" }
]
}