You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by jh...@apache.org on 2015/10/08 09:36:53 UTC
tajo git commit: TAJO-1918: Writing text type in Parquet should
handle text bytes.
Repository: tajo
Updated Branches:
refs/heads/master 21c44800e -> 51198d040
TAJO-1918: Writing text type in Parquet should handle text bytes.
Closes #814
Signed-off-by: Jinho Kim <jh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/51198d04
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/51198d04
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/51198d04
Branch: refs/heads/master
Commit: 51198d0407e868ee788ea36a458642eebcef90c0
Parents: 21c4480
Author: Jongyoung Park <em...@gmail.com>
Authored: Thu Oct 8 16:35:29 2015 +0900
Committer: Jinho Kim <jh...@apache.org>
Committed: Thu Oct 8 16:35:29 2015 +0900
----------------------------------------------------------------------
CHANGES | 3 ++
.../java/org/apache/tajo/datum/DateDatum.java | 5 ++
.../org/apache/tajo/datum/TimestampDatum.java | 5 ++
.../tajo/storage/parquet/TajoWriteSupport.java | 6 +--
.../org/apache/tajo/storage/TestStorages.java | 48 ++++++++++++++++++--
5 files changed, 60 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/51198d04/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 8295b40..23349a7 100644
--- a/CHANGES
+++ b/CHANGES
@@ -333,6 +333,9 @@ Release 0.11.0 - unreleased
BUG FIXES
+ TAJO-1918: Writing text type in Parquet should handle text bytes.
+ (Contributed by Jongyoung Park. Committed by jinho)
+
TAJO-1901: Repair partition throws ArrayIndexOutOfBoundsException
occasionally. (Contributed by jaehwa, committed by hyunsik)
http://git-wip-us.apache.org/repos/asf/tajo/blob/51198d04/tajo-common/src/main/java/org/apache/tajo/datum/DateDatum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/DateDatum.java b/tajo-common/src/main/java/org/apache/tajo/datum/DateDatum.java
index f69aa44..ac84e25 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/DateDatum.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/DateDatum.java
@@ -54,6 +54,11 @@ public class DateDatum extends Datum {
return tm;
}
+ @Override
+ public byte[] asTextBytes() {
+ return asChars().getBytes(TextDatum.DEFAULT_CHARSET);
+ }
+
public int getCenturyOfEra() {
return asTimeMeta().getCenturyOfEra();
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/51198d04/tajo-common/src/main/java/org/apache/tajo/datum/TimestampDatum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/TimestampDatum.java b/tajo-common/src/main/java/org/apache/tajo/datum/TimestampDatum.java
index aaf7beb..5b4c152 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/TimestampDatum.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/TimestampDatum.java
@@ -170,6 +170,11 @@ public class TimestampDatum extends Datum {
}
@Override
+ public byte[] asTextBytes() {
+ return asChars().getBytes(TextDatum.DEFAULT_CHARSET);
+ }
+
+ @Override
public Datum equalsTo(Datum datum) {
if (datum.type() == TajoDataTypes.Type.TIME) {
return timestamp == datum.asInt8() ? BooleanDatum.TRUE : BooleanDatum.FALSE;
http://git-wip-us.apache.org/repos/asf/tajo/blob/51198d04/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoWriteSupport.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoWriteSupport.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoWriteSupport.java
index acb9015..e5ad28c 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoWriteSupport.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoWriteSupport.java
@@ -101,7 +101,7 @@ public class TajoWriteSupport extends WriteSupport<Tuple> {
Type fieldType = fields.get(index);
if (!tuple.isBlankOrNull(tajoIndex)) {
recordConsumer.startField(fieldType.getName(), index);
- writeValue(fieldType, column, tuple, tajoIndex);
+ writeValue(column, tuple, tajoIndex);
recordConsumer.endField(fieldType.getName(), index);
} else if (fieldType.isRepetition(Type.Repetition.REQUIRED)) {
throw new RuntimeException("Null-value for required field: " +
@@ -111,7 +111,7 @@ public class TajoWriteSupport extends WriteSupport<Tuple> {
}
}
- private void writeValue(Type fieldType, Column column, Tuple tuple, int index) {
+ private void writeValue(Column column, Tuple tuple, int index) {
switch (column.getDataType().getType()) {
case BOOLEAN:
recordConsumer.addBoolean(tuple.getBool(index));
@@ -138,7 +138,7 @@ public class TajoWriteSupport extends WriteSupport<Tuple> {
recordConsumer.addBinary(Binary.fromByteArray(tuple.getTextBytes(index)));
break;
case TEXT:
- recordConsumer.addBinary(Binary.fromByteArray(tuple.getBytes(index)));
+ recordConsumer.addBinary(Binary.fromByteArray(tuple.getTextBytes(index)));
break;
case PROTOBUF:
case BLOB:
http://git-wip-us.apache.org/repos/asf/tajo/blob/51198d04/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
index 278de45..dafaf05 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
@@ -706,7 +706,6 @@ public class TestStorages {
appender.init();
QueryId queryid = new QueryId("12345", 5);
- ProtobufDatumFactory factory = ProtobufDatumFactory.get(TajoIdProtos.QueryIdProto.class.getName());
VTuple tuple = new VTuple(new Datum[] {
DatumFactory.createBool(true),
@@ -721,7 +720,7 @@ public class TestStorages {
DatumFactory.createBlob("hyunsik babo".getBytes()),
DatumFactory.createInet4("192.168.0.1"),
NullDatum.get(),
- factory.createDatum(queryid.getProto())
+ ProtobufDatumFactory.createDatum(queryid.getProto())
});
appender.addTuple(tuple);
appender.flush();
@@ -779,7 +778,6 @@ public class TestStorages {
appender.init();
QueryId queryid = new QueryId("12345", 5);
- ProtobufDatumFactory factory = ProtobufDatumFactory.get(TajoIdProtos.QueryIdProto.class.getName());
VTuple tuple = new VTuple(13);
tuple.put(new Datum[] {
@@ -795,7 +793,7 @@ public class TestStorages {
DatumFactory.createBlob("hyunsik babo".getBytes()),
DatumFactory.createInet4("192.168.0.1"),
NullDatum.get(),
- factory.createDatum(queryid.getProto())
+ ProtobufDatumFactory.createDatum(queryid.getProto())
});
appender.addTuple(tuple);
appender.flush();
@@ -1114,4 +1112,46 @@ public class TestStorages {
assertTrue(ok);
}
+
+ @Test
+ public void testDateTextHandling() throws Exception {
+ if (dataFormat.equalsIgnoreCase(BuiltinStorages.AVRO) || internalType) {
+ return;
+ }
+
+ Schema schema = new Schema();
+ schema.addColumn("col1", Type.TEXT);
+
+ KeyValueSet options = new KeyValueSet();
+ TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options);
+
+ FileTablespace sm = TablespaceManager.getLocalFs();
+ Path tablePath = new Path(testDir, "testTextHandling.data");
+
+ Appender appender = sm.getAppender(meta, schema, tablePath);
+
+ appender.init();
+
+ VTuple tuple = new VTuple(1);
+ tuple.put(0, DatumFactory.createDate(1994,7,30));
+
+ appender.addTuple(tuple);
+ appender.flush();
+ appender.close();
+
+ FileStatus status = fs.getFileStatus(tablePath);
+ FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
+ Scanner scanner = sm.getScanner(meta, schema, fragment, null);
+ scanner.init();
+
+ Tuple retrieved;
+ while ((retrieved = scanner.next()) != null) {
+ assertEquals(tuple.get(0).asChars(), retrieved.asDatum(0).asChars());
+ }
+ scanner.close();
+
+ if (internalType){
+ OldStorageManager.clearCache();
+ }
+ }
}
\ No newline at end of file