You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by ji...@apache.org on 2013/08/29 03:38:14 UTC
git commit: TAJO-59: Implement Char Datum Type. (jihoon)
Updated Branches:
refs/heads/master a09ac15ca -> 4e47471c3
TAJO-59: Implement Char Datum Type. (jihoon)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tajo/commit/4e47471c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tajo/tree/4e47471c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tajo/diff/4e47471c
Branch: refs/heads/master
Commit: 4e47471c3c72756821c26c6817611b6cc51c71b4
Parents: a09ac15
Author: Jihoon Son <ji...@apache.org>
Authored: Thu Aug 29 10:37:23 2013 +0900
Committer: Jihoon Son <ji...@apache.org>
Committed: Thu Aug 29 10:37:23 2013 +0900
----------------------------------------------------------------------
CHANGES.txt | 2 +
.../org/apache/tajo/catalog/CatalogUtil.java | 4 +
.../java/org/apache/tajo/catalog/Column.java | 4 +
.../java/org/apache/tajo/catalog/Schema.java | 7 ++
.../org/apache/tajo/catalog/store/DBStore.java | 16 ++--
.../java/org/apache/tajo/datum/CharDatum.java | 84 ++++++++++++--------
.../org/apache/tajo/datum/DatumFactory.java | 14 ++--
.../org/apache/tajo/datum/TestCharDatum.java | 26 +++---
.../main/java/org/apache/tajo/cli/TajoCli.java | 3 +
.../tajo/engine/planner/LogicalPlanner.java | 37 ++++++++-
.../java/org/apache/tajo/storage/CSVFile.java | 11 +--
.../java/org/apache/tajo/storage/RawFile.java | 12 ++-
.../java/org/apache/tajo/storage/RowFile.java | 38 +++------
.../tajo/storage/rcfile/RCFileWrapper.java | 3 +-
.../tajo/storage/trevni/TrevniAppender.java | 4 +-
.../tajo/storage/trevni/TrevniScanner.java | 3 +-
.../org/apache/tajo/storage/TestStorages.java | 13 +--
17 files changed, 178 insertions(+), 103 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/4e47471c/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 03dfaf3..787c0b5 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -3,6 +3,8 @@ Tajo Change Log
Release 0.2.0 - unreleased
NEW FEATURES
+
+ TAJO-59: Implement Char Datum Type. (jihoon)
TAJO-96: Design and implement rewrite rule interface and the rewrite rule
engine. (hyunsik)
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/4e47471c/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
index 3eb08da..8f17852 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
@@ -178,4 +178,8 @@ public class CatalogUtil {
}
return dataTypes;
}
+
+ public static DataType newDataTypeWithLen(Type type, int length) {
+ return DataType.newBuilder().setType(type).setLength(length).build();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/4e47471c/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java
index c215661..2227999 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java
@@ -45,6 +45,10 @@ public class Column implements ProtoObject<ColumnProto>, Cloneable, GsonObject {
public Column(String columnName, TajoDataTypes.Type type) {
this(columnName, CatalogUtil.newDataTypeWithoutLen(type));
}
+
+ public Column(String columnName, TajoDataTypes.Type type, int typeLength) {
+ this(columnName, CatalogUtil.newDataTypeWithLen(type, typeLength));
+ }
public Column(ColumnProto proto) {
this(proto.getColumnName(), proto.getDataType());
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/4e47471c/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java
index 9fee40f..65f6176 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java
@@ -114,9 +114,16 @@ public class Schema implements ProtoObject<SchemaProto>, Cloneable, GsonObject {
}
public synchronized Schema addColumn(String name, Type type) {
+ if (type == Type.CHAR) {
+ return addColumn(name, CatalogUtil.newDataTypeWithLen(type, 1));
+ }
return addColumn(name, CatalogUtil.newDataTypeWithoutLen(type));
}
+ public synchronized Schema addColumn(String name, Type type, int length) {
+ return addColumn(name, CatalogUtil.newDataTypeWithLen(type, length));
+ }
+
public synchronized Schema addColumn(String name, DataType dataType) {
String lowcased = name.toLowerCase();
if(fieldsByName.containsKey(lowcased)) {
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/4e47471c/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/DBStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/DBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/DBStore.java
index 8fb19f0..078ca6e 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/DBStore.java
+++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/DBStore.java
@@ -216,7 +216,7 @@ public class DBStore implements CatalogStore {
+ C_TABLE_ID + " VARCHAR(256) NOT NULL REFERENCES " + TB_TABLES + "("
+ C_TABLE_ID + ") ON DELETE CASCADE, "
+ "column_id INT NOT NULL,"
- + "column_name VARCHAR(256) NOT NULL, " + "data_type CHAR(16), "
+ + "column_name VARCHAR(256) NOT NULL, " + "data_type CHAR(16), " + "type_length INTEGER, "
+ "CONSTRAINT C_COLUMN_ID UNIQUE (" + C_TABLE_ID + ", column_name))";
if (LOG.isDebugEnabled()) {
LOG.debug(columns_ddl);
@@ -439,13 +439,14 @@ public class DBStore implements CatalogStore {
final int columnId, final Column col) {
String sql =
"INSERT INTO " + TB_COLUMNS
- + " (tid, " + C_TABLE_ID + ", column_id, column_name, data_type) "
+ + " (tid, " + C_TABLE_ID + ", column_id, column_name, data_type, type_length) "
+ "VALUES("
+ tid + ","
+ "'" + desc.getName() + "',"
+ columnId + ", "
+ "'" + col.getColumnName() + "',"
- + "'" + col.getDataType().getType().name() + "'"
+ + "'" + col.getDataType().getType().name() + "',"
+ + (col.getDataType().hasLength() ? col.getDataType().getLength() : 0)
+ ")";
return sql;
@@ -615,7 +616,7 @@ public class DBStore implements CatalogStore {
Schema schema = null;
try {
- String sql = "SELECT column_name, data_type from " + TB_COLUMNS
+ String sql = "SELECT column_name, data_type, type_length from " + TB_COLUMNS
+ " WHERE " + C_TABLE_ID + "='" + name + "' ORDER by column_id asc";
stmt = conn.createStatement();
@@ -630,7 +631,12 @@ public class DBStore implements CatalogStore {
+ res.getString("column_name").trim();
Type dataType = getDataType(res.getString("data_type")
.trim());
- schema.addColumn(columnName, dataType);
+ int typeLength = res.getInt("type_length");
+ if (typeLength > 0) {
+ schema.addColumn(columnName, dataType, typeLength);
+ } else {
+ schema.addColumn(columnName, dataType);
+ }
}
} catch (SQLException se) {
throw new IOException(se);
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/4e47471c/tajo-common/src/main/java/org/apache/tajo/datum/CharDatum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/CharDatum.java b/tajo-common/src/main/java/org/apache/tajo/datum/CharDatum.java
index 4d77dad..9ce9089 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/CharDatum.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/CharDatum.java
@@ -21,72 +21,96 @@ package org.apache.tajo.datum;
import com.google.gson.annotations.Expose;
import org.apache.tajo.datum.exception.InvalidOperationException;
+import java.util.Arrays;
+
import static org.apache.tajo.common.TajoDataTypes.Type;
public class CharDatum extends Datum {
- private static final int size = 1;
- @Expose char val;
+ @Expose private int size;
+ @Expose private byte[] bytes;
+ private String chars = null;
public CharDatum() {
super(Type.CHAR);
}
public CharDatum(byte val) {
- this();
- this.val = (char)val;
+ this();
+ this.size = 1;
+ bytes = new byte[size];
+ bytes[0] = val;
}
+ public CharDatum(char val) {
+ this((byte)val);
+ }
+
public CharDatum(byte [] bytes) {
- this(bytes[0]);
+ this();
+ this.bytes = bytes;
+ this.size = bytes.length;
}
- public CharDatum(char val) {
- this();
- this.val = val;
- }
+ public CharDatum(String val) {
+ this(val.getBytes());
+ }
+
+ private String getString() {
+ if (chars == null) {
+ chars = new String(bytes);
+ }
+ return chars;
+ }
@Override
public char asChar() {
- return val;
+ return getString().charAt(0);
}
@Override
- public int asInt4() {
- return val;
- }
+ public short asInt2() {
+ return Short.valueOf(getString());
+ }
@Override
- public long asInt8() {
- return val;
+ public int asInt4() {
+ return Integer.valueOf(getString());
}
@Override
- public byte asByte() {
- return (byte)val;
+ public long asInt8() {
+ return Long.valueOf(getString());
}
@Override
public byte[] asByteArray() {
- byte [] bytes = new byte[1];
- bytes[0] = (byte) val;
return bytes;
}
@Override
public float asFloat4() {
- return val;
+ return Float.valueOf(getString());
}
@Override
public double asFloat8() {
- return val;
+ return Double.valueOf(getString());
}
@Override
+ public byte asByte() {
+ return bytes[0];
+ }
+
+ @Override
public String asChars() {
- return String.valueOf(val);
+ return getString();
}
+ /**
+ * Return the real length of the string
+ * @return the length of the string
+ */
@Override
public int size() {
return size;
@@ -94,14 +118,15 @@ public class CharDatum extends Datum {
@Override
public int hashCode() {
- return val;
+ return getString().hashCode();
}
@Override
public boolean equals(Object obj) {
if (obj instanceof CharDatum) {
CharDatum other = (CharDatum) obj;
- return val == other.val;
+ return this.size == other.size &&
+ Arrays.equals(this.bytes, other.bytes);
}
return false;
@@ -111,7 +136,7 @@ public class CharDatum extends Datum {
public BooleanDatum equalsTo(Datum datum) {
switch (datum.type()) {
case CHAR:
- return DatumFactory.createBool(this.val == (((CharDatum) datum).val));
+ return DatumFactory.createBool(this.equals(datum));
default:
throw new InvalidOperationException(datum.type());
}
@@ -121,13 +146,8 @@ public class CharDatum extends Datum {
public int compareTo(Datum datum) {
switch (datum.type()) {
case CHAR:
- if (val < datum.asChar()) {
- return -1;
- } else if (val > datum.asChar()) {
- return 1;
- } else {
- return 0;
- }
+ CharDatum other = (CharDatum) datum;
+ return this.getString().compareTo(other.getString());
default:
throw new InvalidOperationException(datum.type());
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/4e47471c/tajo-common/src/main/java/org/apache/tajo/datum/DatumFactory.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/DatumFactory.java b/tajo-common/src/main/java/org/apache/tajo/datum/DatumFactory.java
index 453dd4a..72dfc15 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/DatumFactory.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/DatumFactory.java
@@ -23,6 +23,8 @@ import org.apache.tajo.common.TajoDataTypes.DataType;
import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.util.Bytes;
+import java.io.UnsupportedEncodingException;
+
public class DatumFactory {
public static Datum create(DataType type, byte[] val) {
@@ -78,7 +80,7 @@ public class DatumFactory {
case FLOAT8:
return createFloat8(Bytes.toDouble(val));
case CHAR:
- return createChar(val[0]);
+ return createChar(val);
case TEXT:
return createText(val);
case BIT:
@@ -131,12 +133,14 @@ public class DatumFactory {
return new CharDatum(val);
}
- /*
- public static CharDatum createChar(Integer val) {
+ public static CharDatum createChar(byte[] bytes) {
+ return new CharDatum(bytes);
+ }
+
+ public static CharDatum createChar(String val) {
return new CharDatum(val);
}
- */
-
+
public static Int2Datum createInt2(short val) {
return new Int2Datum(val);
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/4e47471c/tajo-common/src/test/java/org/apache/tajo/datum/TestCharDatum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/test/java/org/apache/tajo/datum/TestCharDatum.java b/tajo-common/src/test/java/org/apache/tajo/datum/TestCharDatum.java
index 3e6ed10..1d6da28 100644
--- a/tajo-common/src/test/java/org/apache/tajo/datum/TestCharDatum.java
+++ b/tajo-common/src/test/java/org/apache/tajo/datum/TestCharDatum.java
@@ -34,51 +34,53 @@ public class TestCharDatum {
@Test
public final void testAsInt() {
- Datum d = DatumFactory.createChar((char)5);
+ Datum d = DatumFactory.createChar("5");
assertEquals(5,d.asInt4());
}
@Test
public final void testAsLong() {
- Datum d = DatumFactory.createChar((char)5);
+ Datum d = DatumFactory.createChar("5");
assertEquals(5l,d.asInt8());
}
@Test
public final void testAsByte() {
Datum d = DatumFactory.createChar((char)5);
- assertEquals(5,d.asInt8());
+ assertEquals(5,d.asByte());
}
@Test
public final void testAsFloat() {
- Datum d = DatumFactory.createChar((char)5);
+ Datum d = DatumFactory.createChar("5");
assertTrue(5.0f == d.asFloat4());
}
@Test
public final void testAsDouble() {
- Datum d = DatumFactory.createChar((char)5);
+ Datum d = DatumFactory.createChar("5");
assertTrue(5.0d == d.asFloat8());
}
@Test
public final void testAsChars() {
- Datum d = DatumFactory.createChar((char)5);
- System.out.println(d.asChars());
+ Datum d = DatumFactory.createChar("1234567890");
+ assertEquals("1234567890", d.asChars());
}
@Test
public final void testSize() {
- Datum d = DatumFactory.createChar((char) 1);
- assertEquals(1, d.size());
+ Datum d = DatumFactory.createChar("1234567890".getBytes());
+ assertEquals(10, d.size());
+ d = DatumFactory.createChar("1234567890");
+ assertEquals(10, d.size());
}
@Test
public final void testCompare() {
- Datum a = DatumFactory.createChar('a');
- Datum b = DatumFactory.createChar('b');
- Datum c = DatumFactory.createChar('c');
+ Datum a = DatumFactory.createChar("abc");
+ Datum b = DatumFactory.createChar("abd");
+ Datum c = DatumFactory.createChar("ace");
assertTrue(a.compareTo(b) < 0);
assertTrue(b.compareTo(a) > 0);
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/4e47471c/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/cli/TajoCli.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/cli/TajoCli.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/cli/TajoCli.java
index 55a97ca..083328e 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/cli/TajoCli.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/cli/TajoCli.java
@@ -441,6 +441,9 @@ public class TajoCli {
for(int i = 0; i < desc.getMeta().getSchema().getColumnNum(); i++) {
Column col = desc.getMeta().getSchema().getColumn(i);
sb.append(col.getColumnName()).append("\t").append(col.getDataType().getType());
+ if (col.getDataType().hasLength()) {
+ sb.append("(").append(col.getDataType().getLength()).append(")");
+ }
sb.append("\n");
}
return sb.toString();
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/4e47471c/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
index 4fa4a10..2c4f241 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
@@ -24,6 +24,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
import org.apache.tajo.algebra.*;
+import org.apache.tajo.algebra.CreateTable.ColumnDefinition;
import org.apache.tajo.catalog.*;
import org.apache.tajo.catalog.function.AggFunction;
import org.apache.tajo.catalog.function.GeneralFunction;
@@ -785,16 +786,44 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
private Schema convertTableElementsSchema(CreateTable.ColumnDefinition [] elements) {
Schema schema = new Schema();
- Column column;
for (CreateTable.ColumnDefinition columnDefinition: elements) {
- column = new Column(columnDefinition.getColumnName(),
- TajoDataTypes.Type.valueOf(columnDefinition.getDataType()));
- schema.addColumn(column);
+ schema.addColumn(convertColumn(columnDefinition));
}
return schema;
}
+ private Column convertColumn(ColumnDefinition columnDefinition) {
+ TajoDataTypes.Type type = TajoDataTypes.Type.valueOf(columnDefinition.getDataType());
+ Column column;
+ switch (type) {
+ case CHAR:
+ case VARCHAR:
+ case NCHAR:
+ case NVARCHAR:
+ column = new Column(columnDefinition.getColumnName(),
+ TajoDataTypes.Type.valueOf(columnDefinition.getDataType()),
+ columnDefinition.getLengthOrPrecision());
+ break;
+ case FLOAT4:
+ case FLOAT8:
+ // TODO: support precision
+ column = new Column(columnDefinition.getColumnName(),
+ TajoDataTypes.Type.valueOf(columnDefinition.getDataType()));
+ break;
+ case NUMERIC:
+ case DECIMAL:
+ // TODO: support precision and scale
+ column = new Column(columnDefinition.getColumnName(),
+ TajoDataTypes.Type.valueOf(columnDefinition.getDataType()));
+ break;
+ default:
+ column = new Column(columnDefinition.getColumnName(),
+ TajoDataTypes.Type.valueOf(columnDefinition.getDataType()));
+ }
+ return column;
+ }
+
@Override
public LogicalNode visitDropTable(PlanContext context, Stack<OpType> stack, DropTable dropTable) {
DropTableNode dropTableNode = new DropTableNode(dropTable.getTableName());
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/4e47471c/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/CSVFile.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/CSVFile.java b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/CSVFile.java
index 0896bd1..5c4fa86 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/CSVFile.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/CSVFile.java
@@ -106,11 +106,11 @@ public class CSVFile {
.asByteArray(), false)));
break;
case CHAR:
- sb.append(tuple.getChar(i));
+ CharDatum charDatum = tuple.getChar(i);
+ sb.append(charDatum);
+ byte[] pad = new byte[col.getDataType().getLength()-charDatum.size()];
+ sb.append(new String(pad));
break;
-// case STRING:
-// sb.append(tuple.getString(i));
-// break;
case TEXT:
TextDatum td = tuple.getText(i);
sb.append(td.toString());
@@ -361,7 +361,8 @@ public class CSVFile {
tuple.put(tid, DatumFactory.createBit(Base64.decodeBase64(cell)[0]));
break;
case CHAR:
- tuple.put(tid, DatumFactory.createChar(cell.charAt(0)));
+ String trimmed = cell.trim();
+ tuple.put(tid, DatumFactory.createChar(trimmed));
break;
case BLOB:
tuple.put(tid, DatumFactory.createBlob(Base64.decodeBase64(cell)));
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/4e47471c/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RawFile.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RawFile.java b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RawFile.java
index c8127b1..7c0ea87 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RawFile.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RawFile.java
@@ -35,6 +35,7 @@ import java.io.RandomAccessFile;
import java.net.URI;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
+import java.util.Arrays;
public class RawFile {
public static class RawFileScanner extends FileScanner implements SeekableScanner {
@@ -153,7 +154,11 @@ public class RawFile {
break;
case CHAR :
- tuple.put(i, DatumFactory.createChar(buffer.getChar()));
+ int realLen = buffer.getInt();
+ byte[] buf = new byte[columnTypes[i].getLength()];
+ buffer.get(buf);
+ byte[] charBuf = Arrays.copyOf(buf, realLen);
+ tuple.put(i, DatumFactory.createChar(charBuf));
break;
case INT2 :
@@ -363,7 +368,10 @@ public class RawFile {
break;
case CHAR :
- buffer.putChar(t.get(i).asChar());
+ byte[] src = t.getChar(i).asByteArray();
+ byte[] dst = Arrays.copyOf(src, columnTypes[i].getLength());
+ buffer.putInt(src.length);
+ buffer.put(dst);
break;
case INT2 :
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/4e47471c/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RowFile.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RowFile.java b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RowFile.java
index 6a3bcf1..0911102 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RowFile.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RowFile.java
@@ -211,8 +211,11 @@ public class RowFile {
break;
case CHAR :
- datum = DatumFactory.createChar(buffer.getChar());
- tuple.put(i, datum);
+ int realLen = buffer.getInt();
+ byte[] buf = new byte[col.getDataType().getLength()];
+ buffer.get(buf);
+ byte[] charBuf = Arrays.copyOf(buf, realLen);
+ tuple.put(i, DatumFactory.createChar(charBuf));
break;
case INT2 :
@@ -401,74 +404,51 @@ public class RowFile {
col = schema.getColumn(i);
switch (col.getDataType().getType()) {
case BOOLEAN:
-// out.writeBoolean(t.getByte(i).asBool());
buffer.put(t.getBoolean(i).asByte());
break;
case BIT:
-// out.writeByte(t.getByte(i).asByte());
buffer.put(t.getByte(i).asByte());
break;
case CHAR:
-// out.writeChar(t.getChar(i).asChar());
- buffer.putChar(t.getChar(i).asChar());
+ byte[] src = t.getChar(i).asByteArray();
+ byte[] dst = Arrays.copyOf(src, col.getDataType().getLength());
+ buffer.putInt(src.length);
+ buffer.put(dst);
break;
-// case TEXT:
-// byte[] buf = t.getString(i).asByteArray();
-// if (buf.length > 256) {
-// buf = new byte[256];
-// byte[] str = t.getString(i).asByteArray();
-// System.arraycopy(str, 0, buf, 0, 256);
-// }
-//// out.writeShort(buf.length);
-//// out.write(buf, 0, buf.length);
-// buffer.putShort((short)buf.length);
-// buffer.put(buf, 0, buf.length);
-// break;
case TEXT:
byte [] strbytes = t.getText(i).asByteArray();
buffer.putShort((short)strbytes.length);
buffer.put(strbytes, 0, strbytes.length);
break;
case INT2:
-// out.writeShort(t.getShort(i).asShort());
buffer.putShort(t.getShort(i).asInt2());
break;
case INT4:
-// out.writeInt(t.getInt(i).asInt());
buffer.putInt(t.getInt(i).asInt4());
break;
case INT8:
-// out.writeLong(t.getLong(i).asLong());
buffer.putLong(t.getLong(i).asInt8());
break;
case FLOAT4:
-// out.writeFloat(t.getFloat(i).asFloat());
buffer.putFloat(t.getFloat(i).asFloat4());
break;
case FLOAT8:
-// out.writeDouble(t.getDouble(i).asDouble());
buffer.putDouble(t.getDouble(i).asFloat8());
break;
case BLOB:
byte [] bytes = t.getBytes(i).asByteArray();
-// out.writeInt(bytes.length);
-// out.write(bytes);
buffer.putShort((short)bytes.length);
buffer.put(bytes);
break;
case INET4:
-// out.write(t.getIPv4Bytes(i));
buffer.put(t.getIPv4Bytes(i));
break;
case INET6:
-// out.write(t.getIPv6Bytes(i));
buffer.put(t.getIPv6Bytes(i));
case ARRAY: {
ArrayDatum array = (ArrayDatum) t.get(i);
String json = array.toJson();
byte [] byteArray = json.getBytes();
-// out.writeInt(byteArray.length);
-// out.write(byteArray);
buffer.putShort((short)byteArray.length);
buffer.put(byteArray);
break;
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/4e47471c/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFileWrapper.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFileWrapper.java b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFileWrapper.java
index 92d9acc..6332416 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFileWrapper.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFileWrapper.java
@@ -250,8 +250,9 @@ public class RCFileWrapper {
DatumFactory.createBit(column.get(tid).getBytesCopy()[0]));
break;
case CHAR:
+ byte[] buf = column.get(tid).getBytesCopy();
tuple.put(tid,
- DatumFactory.createChar(column.get(tid).getBytesCopy()[0]));
+ DatumFactory.createChar(buf));
break;
case INT2:
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/4e47471c/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniAppender.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniAppender.java b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniAppender.java
index 11dcfbf..eef95e0 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniAppender.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniAppender.java
@@ -89,7 +89,7 @@ public class TrevniAppender extends FileAppender {
case BIT:
return ValueType.INT;
case CHAR:
- return ValueType.INT;
+ return ValueType.STRING;
case INT2:
return ValueType.INT;
case INT4:
@@ -134,7 +134,6 @@ public class TrevniAppender extends FileAppender {
switch (col.getDataType().getType()) {
case BOOLEAN:
case BIT:
- case CHAR:
case INT2:
case INT4:
writer.writeValue(t.get(i).asInt4(), i);
@@ -148,6 +147,7 @@ public class TrevniAppender extends FileAppender {
case FLOAT8:
writer.writeValue(t.get(i).asFloat8(), i);
break;
+ case CHAR:
case TEXT:
writer.writeValue(t.get(i).asChars(), i);
break;
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/4e47471c/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniScanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniScanner.java b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniScanner.java
index ef43671..be23d48 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniScanner.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniScanner.java
@@ -93,8 +93,9 @@ public class TrevniScanner extends FileScanner {
DatumFactory.createBit(((Integer) columns[i].nextValue()).byteValue()));
break;
case CHAR:
+ String str = (String) columns[i].nextValue();
tuple.put(tid,
- DatumFactory.createChar(((Integer)columns[i].nextValue()).byteValue()));
+ DatumFactory.createChar(str));
break;
case INT2:
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/4e47471c/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/TestStorages.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/TestStorages.java b/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/TestStorages.java
index 0bc2a80..22207fe 100644
--- a/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/TestStorages.java
+++ b/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/TestStorages.java
@@ -185,7 +185,7 @@ public class TestStorages {
Schema schema = new Schema();
schema.addColumn("col1", Type.BOOLEAN);
schema.addColumn("col2", Type.BIT);
- schema.addColumn("col3", Type.CHAR);
+ schema.addColumn("col3", Type.CHAR, 7);
schema.addColumn("col4", Type.INT2);
schema.addColumn("col5", Type.INT4);
schema.addColumn("col6", Type.INT8);
@@ -206,7 +206,7 @@ public class TestStorages {
tuple.put(new Datum[] {
DatumFactory.createBool(true),
DatumFactory.createBit((byte) 0x99),
- DatumFactory.createChar('7'),
+ DatumFactory.createChar("hyunsik"),
DatumFactory.createInt2((short) 17),
DatumFactory.createInt4(59),
DatumFactory.createInt8(23l),
@@ -225,9 +225,12 @@ public class TestStorages {
Fragment fragment = new Fragment("table", tablePath, meta, 0, status.getLen());
Scanner scanner = StorageManager.getScanner(conf, meta, fragment);
scanner.init();
- Tuple retrieved = scanner.next();
- for (int i = 0; i < tuple.size(); i++) {
- assertEquals(tuple.get(i), retrieved.get(i));
+
+ Tuple retrieved;
+ while ((retrieved=scanner.next()) != null) {
+ for (int i = 0; i < tuple.size(); i++) {
+ assertEquals(tuple.get(i), retrieved.get(i));
+ }
}
}
}