You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by bl...@apache.org on 2016/06/25 14:47:00 UTC
tajo git commit: TAJO-2164: SequenceFile print wrong values with
TextSerializerDeserializer.
Repository: tajo
Updated Branches:
refs/heads/master 740720a2f -> 8dbb8ca22
TAJO-2164: SequenceFile print wrong values with TextSerializerDeserializer.
Closes #1033
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/8dbb8ca2
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/8dbb8ca2
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/8dbb8ca2
Branch: refs/heads/master
Commit: 8dbb8ca22520a0fd350ada2dd8a0c75294e0bc59
Parents: 740720a
Author: JaeHwa Jung <bl...@apache.org>
Authored: Sat Jun 25 23:46:01 2016 +0900
Committer: JaeHwa Jung <bl...@apache.org>
Committed: Sat Jun 25 23:46:01 2016 +0900
----------------------------------------------------------------------
CHANGES | 3 +
.../org/apache/tajo/catalog/CatalogUtil.java | 2 +-
.../tajo/catalog/store/HiveCatalogStore.java | 14 +-
.../catalog/store/TestHiveCatalogStore.java | 6 +
.../planner/physical/PhysicalPlanUtil.java | 9 +-
.../sequencefile/SequenceFileAppender.java | 23 +++-
.../sequencefile/SequenceFileScanner.java | 36 +++--
.../apache/tajo/storage/text/TextLineSerDe.java | 8 +-
.../org/apache/tajo/storage/TestStorages.java | 134 +++++++++++++++++++
9 files changed, 207 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/8dbb8ca2/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 7c27422..4903dd4 100644
--- a/CHANGES
+++ b/CHANGES
@@ -151,6 +151,9 @@ Release 0.12.0 - unreleased
BUG FIXES
+ TAJO-2164: SequenceFile print wrong values with TextSerializerDeserializer.
+ (jaehwa)
+
TAJO-2163: No error message is showed when query execution failed. (Wonjun Hong via jihoon)
TAJO-2150: Fix invalid implementation of date time in jdbc. (jinho)
http://git-wip-us.apache.org/repos/asf/tajo/blob/8dbb8ca2/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
index f15ce03..9bda01b 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
@@ -768,7 +768,7 @@ public class CatalogUtil {
options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE);
} else if (dataFormat.equalsIgnoreCase("SEQUENCEFILE")) {
options.set(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE);
- options.set(StorageConstants.SEQUENCEFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
+ options.set(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
} else if (dataFormat.equalsIgnoreCase("PARQUET")) {
options.set(BLOCK_SIZE, StorageConstants.PARQUET_DEFAULT_BLOCK_SIZE);
options.set(PAGE_SIZE, StorageConstants.PARQUET_DEFAULT_PAGE_SIZE);
http://git-wip-us.apache.org/repos/asf/tajo/blob/8dbb8ca2/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java
index b855c77..1a154d6 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java
@@ -219,8 +219,8 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore {
}
} else if (BuiltinStorages.SEQUENCE_FILE.equals(dataFormat)) {
- options.set(StorageConstants.SEQUENCEFILE_DELIMITER, StringEscapeUtils.escapeJava(fieldDelimiter));
- options.set(StorageConstants.SEQUENCEFILE_NULL, StringEscapeUtils.escapeJava(nullFormat));
+ options.set(StorageConstants.TEXT_DELIMITER, StringEscapeUtils.escapeJava(fieldDelimiter));
+ options.set(StorageConstants.TEXT_NULL, StringEscapeUtils.escapeJava(nullFormat));
String serde = properties.getProperty(serdeConstants.SERIALIZATION_LIB);
if (LazyBinarySerDe.class.getName().equals(serde)) {
options.set(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE);
@@ -527,7 +527,7 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore {
if (StorageConstants.DEFAULT_TEXT_SERDE.equals(serde)) {
sd.getSerdeInfo().setSerializationLib(LazySimpleSerDe.class.getName());
- String fieldDelimiter = tableDesc.getMeta().getProperty(StorageConstants.SEQUENCEFILE_DELIMITER,
+ String fieldDelimiter = tableDesc.getMeta().getProperty(StorageConstants.TEXT_DELIMITER,
StorageConstants.DEFAULT_FIELD_DELIMITER);
// User can use an unicode for filed delimiter such as \u0001, \001.
@@ -539,15 +539,15 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore {
StringEscapeUtils.unescapeJava(fieldDelimiter));
sd.getSerdeInfo().putToParameters(serdeConstants.FIELD_DELIM,
StringEscapeUtils.unescapeJava(fieldDelimiter));
- table.getParameters().remove(StorageConstants.SEQUENCEFILE_DELIMITER);
+ table.getParameters().remove(StorageConstants.TEXT_DELIMITER);
} else {
sd.getSerdeInfo().setSerializationLib(LazyBinarySerDe.class.getName());
}
- if (tableDesc.getMeta().containsProperty(StorageConstants.SEQUENCEFILE_NULL)) {
+ if (tableDesc.getMeta().containsProperty(StorageConstants.TEXT_NULL)) {
table.putToParameters(serdeConstants.SERIALIZATION_NULL_FORMAT,
- StringEscapeUtils.unescapeJava(tableDesc.getMeta().getProperty(StorageConstants.SEQUENCEFILE_NULL)));
- table.getParameters().remove(StorageConstants.SEQUENCEFILE_NULL);
+ StringEscapeUtils.unescapeJava(tableDesc.getMeta().getProperty(StorageConstants.TEXT_NULL)));
+ table.getParameters().remove(StorageConstants.TEXT_NULL);
}
} else if (tableDesc.getMeta().getDataFormat().equalsIgnoreCase(BuiltinStorages.PARQUET)) {
StorageFormatDescriptor descriptor = storageFormatFactory.get(IOConstants.PARQUET);
http://git-wip-us.apache.org/repos/asf/tajo/blob/8dbb8ca2/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java
index 1260371..d1be35b 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java
@@ -36,6 +36,7 @@ import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.catalog.proto.CatalogProtos.PartitionKeyProto;
import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.datum.NullDatum;
import org.apache.tajo.schema.IdentifierUtil;
import org.apache.tajo.storage.StorageConstants;
import org.apache.tajo.util.CommonTestingUtil;
@@ -595,6 +596,8 @@ public class TestHiveCatalogStore {
public void testTableUsingSequenceFileWithTextSerde() throws Exception {
KeyValueSet options = new KeyValueSet();
options.set(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE);
+ options.set(StorageConstants.TEXT_DELIMITER, "\u0001");
+ options.set(StorageConstants.TEXT_NULL, NullDatum.DEFAULT_TEXT);
TableMeta meta = new TableMeta(BuiltinStorages.SEQUENCE_FILE, options);
org.apache.tajo.catalog.Schema schema = SchemaBuilder.builder()
@@ -622,6 +625,9 @@ public class TestHiveCatalogStore {
}
assertEquals(StorageConstants.DEFAULT_TEXT_SERDE, table1.getMeta().getProperty(StorageConstants.SEQUENCEFILE_SERDE));
+ assertEquals("\u0001", StringEscapeUtils.unescapeJava(table1.getMeta().getProperty(StorageConstants
+ .TEXT_DELIMITER)));
+ assertEquals(NullDatum.DEFAULT_TEXT, table1.getMeta().getProperty(StorageConstants.TEXT_NULL));
store.dropTable(DB_NAME, REGION);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/8dbb8ca2/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java
index 074d0ab..42ada8b 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java
@@ -216,7 +216,7 @@ public class PhysicalPlanUtil {
} else if (dataFormat.equalsIgnoreCase(BuiltinStorages.RCFILE)) {
meta.putProperty(StorageConstants.RCFILE_NULL, nullChar);
} else if (dataFormat.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) {
- meta.putProperty(StorageConstants.SEQUENCEFILE_NULL, nullChar);
+ meta.putProperty(StorageConstants.TEXT_NULL, nullChar);
}
}
@@ -233,7 +233,12 @@ public class PhysicalPlanUtil {
} else if (dataFormat.equalsIgnoreCase(BuiltinStorages.RCFILE)) {
return meta.containsProperty(StorageConstants.RCFILE_NULL);
} else if (dataFormat.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) {
- return meta.containsProperty(StorageConstants.SEQUENCEFILE_NULL);
+ if (!meta.containsProperty(StorageConstants.TEXT_NULL)
+ && meta.containsProperty(StorageConstants.SEQUENCEFILE_NULL)) {
+ return meta.containsProperty(StorageConstants.SEQUENCEFILE_NULL);
+ } else {
+ return meta.containsProperty(StorageConstants.TEXT_NULL);
+ }
} else {
return false;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/8dbb8ca2/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java
index 2bb3e21..5d8aa83 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java
@@ -83,17 +83,34 @@ public class SequenceFileAppender extends FileAppender {
this.fs = path.getFileSystem(conf);
- this.delimiter = StringEscapeUtils.unescapeJava(this.meta.getProperty(StorageConstants.SEQUENCEFILE_DELIMITER,
+ // Set value of non-deprecated key for backward compatibility.
+ if (!meta.containsProperty(StorageConstants.TEXT_DELIMITER)
+ && meta.containsProperty(StorageConstants.SEQUENCEFILE_DELIMITER)) {
+ this.delimiter = StringEscapeUtils.unescapeJava(meta.getProperty(StorageConstants.SEQUENCEFILE_DELIMITER,
StorageConstants.DEFAULT_FIELD_DELIMITER)).charAt(0);
- this.columnNum = schema.size();
- String nullCharacters = StringEscapeUtils.unescapeJava(this.meta.getProperty(StorageConstants.SEQUENCEFILE_NULL,
+ } else {
+ this.delimiter = StringEscapeUtils.unescapeJava(meta.getProperty(StorageConstants.TEXT_DELIMITER,
+ StorageConstants.DEFAULT_FIELD_DELIMITER)).charAt(0);
+ }
+
+ String nullCharacters;
+ if (!meta.containsProperty(StorageConstants.TEXT_NULL)
+ && meta.containsProperty(StorageConstants.SEQUENCEFILE_NULL)) {
+ nullCharacters = StringEscapeUtils.unescapeJava(meta.getProperty(StorageConstants.SEQUENCEFILE_NULL,
NullDatum.DEFAULT_TEXT));
+ } else {
+ nullCharacters = StringEscapeUtils.unescapeJava(meta.getProperty(StorageConstants.TEXT_NULL,
+ NullDatum.DEFAULT_TEXT));
+ }
+
if (StringUtils.isEmpty(nullCharacters)) {
nullChars = NullDatum.get().asTextBytes();
} else {
nullChars = nullCharacters.getBytes();
}
+ this.columnNum = schema.size();
+
if(this.meta.containsProperty(StorageConstants.COMPRESSION_CODEC)) {
String codecName = this.meta.getProperty(StorageConstants.COMPRESSION_CODEC);
codecFactory = new CompressionCodecFactory(conf);
http://git-wip-us.apache.org/repos/asf/tajo/blob/8dbb8ca2/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java
index b7dc1ec..491a9a8 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java
@@ -32,6 +32,7 @@ import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.datum.Datum;
import org.apache.tajo.datum.NullDatum;
+import org.apache.tajo.exception.TajoInternalError;
import org.apache.tajo.exception.TajoRuntimeException;
import org.apache.tajo.exception.UnsupportedException;
import org.apache.tajo.plan.expr.EvalNode;
@@ -96,7 +97,28 @@ public class SequenceFileScanner extends FileScanner {
reader = new SequenceFile.Reader(fs, fragment.getPath(), conf);
- String nullCharacters = StringEscapeUtils.unescapeJava(this.meta.getProperty(StorageConstants.SEQUENCEFILE_NULL,
+ // Set value of non-deprecated key for backward compatibility.
+ TableMeta tableMeta;
+ try {
+ tableMeta = (TableMeta) meta.clone();
+
+ if (!tableMeta.containsProperty(StorageConstants.TEXT_DELIMITER)) {
+ tableMeta.putProperty(StorageConstants.TEXT_DELIMITER, tableMeta.getProperty(StorageConstants
+ .SEQUENCEFILE_DELIMITER));
+ }
+
+ if (!tableMeta.containsProperty(StorageConstants.TEXT_NULL) && tableMeta.containsProperty(StorageConstants
+ .SEQUENCEFILE_NULL)) {
+ tableMeta.putProperty(StorageConstants.TEXT_NULL, tableMeta.getProperty(StorageConstants.SEQUENCEFILE_NULL));
+ }
+ } catch (CloneNotSupportedException e) {
+ throw new TajoInternalError(e);
+ }
+
+ String delim = tableMeta.getProperty(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
+ this.delimiter = StringEscapeUtils.unescapeJava(delim).charAt(0);
+
+ String nullCharacters = StringEscapeUtils.unescapeJava(tableMeta.getProperty(StorageConstants.TEXT_NULL,
NullDatum.DEFAULT_TEXT));
if (StringUtils.isEmpty(nullCharacters)) {
nullChars = NullDatum.get().asTextBytes();
@@ -104,9 +126,6 @@ public class SequenceFileScanner extends FileScanner {
nullChars = nullCharacters.getBytes();
}
- String delim = meta.getProperty(StorageConstants.SEQUENCEFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
- this.delimiter = StringEscapeUtils.unescapeJava(delim).charAt(0);
-
this.start = fragment.getStartKey();
this.end = start + fragment.getLength();
@@ -120,9 +139,6 @@ public class SequenceFileScanner extends FileScanner {
}
outTuple = new VTuple(targets.length);
- deserializer = DelimitedTextFile.getLineSerde(meta).createDeserializer(schema, meta, targets);
- deserializer.init();
-
fieldIsNull = new boolean[schema.getRootColumns().size()];
fieldStart = new int[schema.getRootColumns().size()];
fieldLength = new int[schema.getRootColumns().size()];
@@ -130,12 +146,16 @@ public class SequenceFileScanner extends FileScanner {
prepareProjection(targets);
try {
- String serdeClass = this.meta.getProperty(StorageConstants.SEQUENCEFILE_SERDE, TextSerializerDeserializer.class.getName());
+ String serdeClass = tableMeta.getProperty(StorageConstants.SEQUENCEFILE_SERDE,
+ TextSerializerDeserializer.class.getName());
serde = (SerializerDeserializer) Class.forName(serdeClass).newInstance();
serde.init(schema);
if (serde instanceof BinarySerializerDeserializer) {
hasBinarySerDe = true;
+ } else {
+ deserializer = DelimitedTextFile.getLineSerde(tableMeta).createDeserializer(schema, tableMeta, targets);
+ deserializer.init();
}
Class<? extends Writable> keyClass = (Class<? extends Writable>)Class.forName(reader.getKeyClassName());
http://git-wip-us.apache.org/repos/asf/tajo/blob/8dbb8ca2/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextLineSerDe.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextLineSerDe.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextLineSerDe.java
index 94a0ba0..36183b7 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextLineSerDe.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextLineSerDe.java
@@ -43,13 +43,7 @@ public abstract class TextLineSerDe {
public abstract TextLineSerializer createSerializer(Schema schema, TableMeta meta);
public static ByteBuf getNullChars(TableMeta meta) {
- byte[] nullCharByteArray;
- if (meta.getDataFormat().equals(BuiltinStorages.SEQUENCE_FILE)) {
- nullCharByteArray = getNullCharsAsBytes(meta, StorageConstants.SEQUENCEFILE_NULL, "\\");
- } else {
- nullCharByteArray = getNullCharsAsBytes(meta);
- }
-
+ byte[] nullCharByteArray = getNullCharsAsBytes(meta);
ByteBuf nullChars = BufferPool.directBuffer(nullCharByteArray.length, nullCharByteArray.length);
nullChars.writeBytes(nullCharByteArray);
http://git-wip-us.apache.org/repos/asf/tajo/blob/8dbb8ca2/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
index c1423d7..deb758d 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
@@ -867,6 +867,140 @@ public class TestStorages {
}
@Test
+ public void testSequenceFileTextSerializeDeserializeWithDeprecatedProperties() throws IOException {
+ if(!dataFormat.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) return;
+
+ Schema schema = SchemaBuilder.builder()
+ .add("col1", Type.BOOLEAN)
+ .add("col2", CatalogUtil.newDataTypeWithLen(Type.CHAR, 7))
+ .add("col3", Type.INT2)
+ .add("col4", Type.INT4)
+ .add("col5", Type.INT8)
+ .add("col6", Type.FLOAT4)
+ .add("col7", Type.FLOAT8)
+ .add("col8", Type.TEXT)
+ .add("col9", Type.BLOB)
+ .add("col10", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())).build();
+
+ TableMeta meta = CatalogUtil.newTableMeta(dataFormat, conf);
+ meta.putProperty(StorageConstants.SEQUENCEFILE_SERDE, TextSerializerDeserializer.class.getName());
+ meta.putProperty(StorageConstants.SEQUENCEFILE_DELIMITER, "\u0001");
+ meta.putProperty(StorageConstants.SEQUENCEFILE_NULL, "\\");
+
+ Path tablePath = new Path(testDir, "testSequenceFileTextSerializeDeserializeWithDeprecatedProperties.data");
+ FileTablespace sm = TablespaceManager.getLocalFs();
+ Appender appender = sm.getAppender(meta, schema, tablePath);
+ appender.enableStats();
+ appender.init();
+
+ QueryId queryid = new QueryId("12345", 5);
+
+ VTuple tuple = new VTuple(new Datum[] {
+ DatumFactory.createBool(true),
+ DatumFactory.createChar("jinho"),
+ DatumFactory.createInt2((short) 17),
+ DatumFactory.createInt4(59),
+ DatumFactory.createInt8(23l),
+ DatumFactory.createFloat4(77.9f),
+ DatumFactory.createFloat8(271.9f),
+ DatumFactory.createText("jinho"),
+ DatumFactory.createBlob("hyunsik babo".getBytes()),
+ ProtobufDatumFactory.createDatum(queryid.getProto())
+ });
+ appender.addTuple(tuple);
+ appender.flush();
+ appender.close();
+
+ FileStatus status = fs.getFileStatus(tablePath);
+ assertEquals(appender.getStats().getNumBytes().longValue(), status.getLen());
+
+ FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
+ Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment, null);
+ scanner.init();
+
+ assertTrue(scanner instanceof SequenceFileScanner);
+ Writable key = ((SequenceFileScanner) scanner).getKey();
+ assertEquals(key.getClass().getCanonicalName(), LongWritable.class.getCanonicalName());
+
+ Tuple retrieved;
+ while ((retrieved=scanner.next()) != null) {
+ for (int i = 0; i < tuple.size(); i++) {
+ assertEquals(tuple.get(i), retrieved.asDatum(i));
+ }
+ }
+ scanner.close();
+ assertEquals(appender.getStats().getNumBytes().longValue(), scanner.getInputStats().getNumBytes().longValue());
+ assertEquals(appender.getStats().getNumRows().longValue(), scanner.getInputStats().getNumRows().longValue());
+ }
+
+ @Test
+ public void testSequenceFileTextSerializeDeserializeWithNonDeprecatedProperties() throws IOException {
+ if(!dataFormat.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) return;
+
+ Schema schema = SchemaBuilder.builder()
+ .add("col1", Type.BOOLEAN)
+ .add("col2", CatalogUtil.newDataTypeWithLen(Type.CHAR, 7))
+ .add("col3", Type.INT2)
+ .add("col4", Type.INT4)
+ .add("col5", Type.INT8)
+ .add("col6", Type.FLOAT4)
+ .add("col7", Type.FLOAT8)
+ .add("col8", Type.TEXT)
+ .add("col9", Type.BLOB)
+ .add("col10", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())).build();
+
+ TableMeta meta = CatalogUtil.newTableMeta(dataFormat, conf);
+ meta.putProperty(StorageConstants.SEQUENCEFILE_SERDE, TextSerializerDeserializer.class.getName());
+ meta.putProperty(StorageConstants.TEXT_DELIMITER, "\u0001");
+ meta.putProperty(StorageConstants.TEXT_NULL, "\\");
+
+ Path tablePath = new Path(testDir, "testSequenceFileTextSerializeDeserializeWithNonDeprecatedProperties.data");
+ FileTablespace sm = TablespaceManager.getLocalFs();
+ Appender appender = sm.getAppender(meta, schema, tablePath);
+ appender.enableStats();
+ appender.init();
+
+ QueryId queryid = new QueryId("12345", 5);
+
+ VTuple tuple = new VTuple(new Datum[] {
+ DatumFactory.createBool(true),
+ DatumFactory.createChar("jinho"),
+ DatumFactory.createInt2((short) 17),
+ DatumFactory.createInt4(59),
+ DatumFactory.createInt8(23l),
+ DatumFactory.createFloat4(77.9f),
+ DatumFactory.createFloat8(271.9f),
+ DatumFactory.createText("jinho"),
+ DatumFactory.createBlob("hyunsik babo".getBytes()),
+ ProtobufDatumFactory.createDatum(queryid.getProto())
+ });
+ appender.addTuple(tuple);
+ appender.flush();
+ appender.close();
+
+ FileStatus status = fs.getFileStatus(tablePath);
+ assertEquals(appender.getStats().getNumBytes().longValue(), status.getLen());
+
+ FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
+ Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment, null);
+ scanner.init();
+
+ assertTrue(scanner instanceof SequenceFileScanner);
+ Writable key = ((SequenceFileScanner) scanner).getKey();
+ assertEquals(key.getClass().getCanonicalName(), LongWritable.class.getCanonicalName());
+
+ Tuple retrieved;
+ while ((retrieved=scanner.next()) != null) {
+ for (int i = 0; i < tuple.size(); i++) {
+ assertEquals(tuple.get(i), retrieved.asDatum(i));
+ }
+ }
+ scanner.close();
+ assertEquals(appender.getStats().getNumBytes().longValue(), scanner.getInputStats().getNumBytes().longValue());
+ assertEquals(appender.getStats().getNumRows().longValue(), scanner.getInputStats().getNumRows().longValue());
+ }
+
+ @Test
public void testSequenceFileBinarySerializeDeserialize() throws IOException {
if(!dataFormat.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) return;