You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by jh...@apache.org on 2014/01/30 09:54:00 UTC
git commit: TAJO-506: RawFile cannot support DATE type. (jinho)
Updated Branches:
refs/heads/master 8aa0e55d7 -> 567183fc2
TAJO-506: RawFile cannot support DATE type. (jinho)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tajo/commit/567183fc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tajo/tree/567183fc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tajo/diff/567183fc
Branch: refs/heads/master
Commit: 567183fc20e10d56390fd1b14052d082c6ef4623
Parents: 8aa0e55
Author: jinossy <ji...@gmail.com>
Authored: Thu Jan 30 17:52:48 2014 +0900
Committer: jinossy <ji...@gmail.com>
Committed: Thu Jan 30 17:52:48 2014 +0900
----------------------------------------------------------------------
CHANGES.txt | 2 +
.../org/apache/tajo/datum/TimestampDatum.java | 5 +++
.../apache/tajo/datum/TestTimestampDatum.java | 5 ++-
.../engine/planner/RangePartitionAlgorithm.java | 7 +++
.../apache/tajo/engine/query/TestSortQuery.java | 10 +++++
.../resources/dataset/TestSortQuery/table1.tbl | 5 +++
.../create_table_with_date_ddl.sql | 10 +++++
.../queries/TestSortQuery/testSortWithDate.sql | 1 +
.../TestSortQuery/testSortWithDate.result | 7 +++
.../java/org/apache/tajo/storage/RawFile.java | 36 ++++++++++-----
.../org/apache/tajo/storage/RowStoreUtil.java | 6 ++-
.../org/apache/tajo/storage/StorageUtil.java | 47 ++++++++++++++------
.../org/apache/tajo/storage/TestStorages.java | 43 ++++++++++++++++++
13 files changed, 156 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/567183fc/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 0fe9dc5..50b3695 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -236,6 +236,8 @@ Release 0.8.0 - unreleased
BUG FIXES
+ TAJO-506: RawFile cannot support DATE type. (jinho)
+
TAJO-566: BIN/TAJO_DUMP makes wrong ddl script. (hyoungjunkim via hyunsik)
TAJO-567: Expression projection bugs. (hyunsik)
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/567183fc/tajo-common/src/main/java/org/apache/tajo/datum/TimestampDatum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/TimestampDatum.java b/tajo-common/src/main/java/org/apache/tajo/datum/TimestampDatum.java
index 3ff1fca..c1fad56 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/TimestampDatum.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/TimestampDatum.java
@@ -123,6 +123,11 @@ public class TimestampDatum extends Datum {
return dateTime.getWeekOfWeekyear();
}
+ @Override
+ public long asInt8() {
+ return dateTime.getMillis();
+ }
+
public String toString() {
return asChars();
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/567183fc/tajo-common/src/test/java/org/apache/tajo/datum/TestTimestampDatum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/test/java/org/apache/tajo/datum/TestTimestampDatum.java b/tajo-common/src/test/java/org/apache/tajo/datum/TestTimestampDatum.java
index 0482857..4a6b181 100644
--- a/tajo-common/src/test/java/org/apache/tajo/datum/TestTimestampDatum.java
+++ b/tajo-common/src/test/java/org/apache/tajo/datum/TestTimestampDatum.java
@@ -48,10 +48,11 @@ public class TestTimestampDatum {
d.asInt4();
}
- @Test(expected = InvalidCastException.class)
+ @Test
public final void testAsInt8() {
Datum d = DatumFactory.createTimeStamp(timestamp);
- assertEquals(timestamp, d.asInt8());
+ long javaTime = timestamp * 1000l;
+ assertEquals(javaTime, d.asInt8());
}
@Test(expected = InvalidCastException.class)
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/567183fc/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/RangePartitionAlgorithm.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/RangePartitionAlgorithm.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/RangePartitionAlgorithm.java
index 35d7743..fc56b55 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/RangePartitionAlgorithm.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/RangePartitionAlgorithm.java
@@ -87,6 +87,13 @@ public abstract class RangePartitionAlgorithm {
case TEXT:
columnCard = new BigDecimal(end.asChars().charAt(0) - start.asChars().charAt(0));
break;
+ case DATE:
+ columnCard = new BigDecimal(end.asInt4() - start.asInt4());
+ break;
+ case TIME:
+ case TIMESTAMP:
+ columnCard = new BigDecimal(end.asInt8() - start.asInt8());
+ break;
default:
throw new UnsupportedOperationException(dataType + " is not supported yet");
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/567183fc/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java
index 8601cb5..5e5c70c 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java
@@ -93,4 +93,14 @@ public class TestSortQuery extends QueryTestCaseBase {
assertResultSet(res);
cleanupQuery(res);
}
+
+ @Test
+ public final void testSortWithDate() throws Exception {
+ //select col1, col2, l_comment from table1 order by col1, col2;
+ executeDDL("create_table_with_date_ddl.sql", "table1.tbl");
+
+ ResultSet res = executeQuery();
+ assertResultSet(res);
+ cleanupQuery(res);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/567183fc/tajo-core/tajo-core-backend/src/test/resources/dataset/TestSortQuery/table1.tbl
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/resources/dataset/TestSortQuery/table1.tbl b/tajo-core/tajo-core-backend/src/test/resources/dataset/TestSortQuery/table1.tbl
new file mode 100644
index 0000000..a1ebde3
--- /dev/null
+++ b/tajo-core/tajo-core-backend/src/test/resources/dataset/TestSortQuery/table1.tbl
@@ -0,0 +1,5 @@
+1997-11-09 20:34:56|1996-04-12|15:34:56
+1997-11-09 20:34:56|1996-03-13|19:34:56
+1993-11-09 20:34:56|1997-01-28|08:34:56
+1995-11-09 20:34:56|1994-02-02|17:34:56
+1995-11-09 20:34:56|1993-11-09|20:34:56
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/567183fc/tajo-core/tajo-core-backend/src/test/resources/queries/TestSortQuery/create_table_with_date_ddl.sql
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/resources/queries/TestSortQuery/create_table_with_date_ddl.sql b/tajo-core/tajo-core-backend/src/test/resources/queries/TestSortQuery/create_table_with_date_ddl.sql
new file mode 100644
index 0000000..846cbb6
--- /dev/null
+++ b/tajo-core/tajo-core-backend/src/test/resources/queries/TestSortQuery/create_table_with_date_ddl.sql
@@ -0,0 +1,10 @@
+-- Sort Table
+-- It is used in TestSortQuery::testSortWithDate
+
+create external table table1 (
+ col1 timestamp,
+ col2 date,
+ col3 time
+) using csv
+with ('csvfile.delimiter'='|', 'csvfile.null'='NULL')
+location ${table.path};
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/567183fc/tajo-core/tajo-core-backend/src/test/resources/queries/TestSortQuery/testSortWithDate.sql
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/resources/queries/TestSortQuery/testSortWithDate.sql b/tajo-core/tajo-core-backend/src/test/resources/queries/TestSortQuery/testSortWithDate.sql
new file mode 100644
index 0000000..490e8d0
--- /dev/null
+++ b/tajo-core/tajo-core-backend/src/test/resources/queries/TestSortQuery/testSortWithDate.sql
@@ -0,0 +1 @@
+select col1, col2, col3 from table1 order by col1, col2, col3;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/567183fc/tajo-core/tajo-core-backend/src/test/resources/results/TestSortQuery/testSortWithDate.result
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/resources/results/TestSortQuery/testSortWithDate.result b/tajo-core/tajo-core-backend/src/test/resources/results/TestSortQuery/testSortWithDate.result
new file mode 100644
index 0000000..118909c
--- /dev/null
+++ b/tajo-core/tajo-core-backend/src/test/resources/results/TestSortQuery/testSortWithDate.result
@@ -0,0 +1,7 @@
+col1,col2,col3
+-------------------------------
+1993-11-09 20:34:56,1997-01-28,08:34:56
+1995-11-09 20:34:56,1993-11-09,20:34:56
+1995-11-09 20:34:56,1994-02-02,17:34:56
+1997-11-09 20:34:56,1996-03-13,19:34:56
+1997-11-09 20:34:56,1996-04-12,15:34:56
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/567183fc/tajo-storage/src/main/java/org/apache/tajo/storage/RawFile.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/RawFile.java b/tajo-storage/src/main/java/org/apache/tajo/storage/RawFile.java
index db511dc..749b2e4 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/RawFile.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/RawFile.java
@@ -30,7 +30,6 @@ import org.apache.tajo.common.TajoDataTypes.DataType;
import org.apache.tajo.datum.DatumFactory;
import org.apache.tajo.datum.NullDatum;
import org.apache.tajo.datum.ProtobufDatumFactory;
-import org.apache.tajo.datum.TimestampDatum;
import org.apache.tajo.storage.fragment.FileFragment;
import org.apache.tajo.util.BitArray;
@@ -212,10 +211,6 @@ public class RawFile {
tuple.put(i, DatumFactory.createText(new String(strBytes2)));
break;
- case TIMESTAMP:
- tuple.put(i, DatumFactory.createTimeStampFromMillis(buffer.getLong()));
- break;
-
case BLOB : {
//byte [] rawBytes = getColumnBytes();
int byteSize = buffer.getInt();
@@ -244,6 +239,25 @@ public class RawFile {
tuple.put(i, DatumFactory.createInet4(ipv4Bytes));
break;
+ case DATE: {
+ int val = buffer.getInt();
+ if (val < Integer.MIN_VALUE + 1) {
+ tuple.put(i, DatumFactory.createNullDatum());
+ } else {
+ tuple.put(i, DatumFactory.createFromInt4(columnTypes[i], val));
+ }
+ break;
+ }
+ case TIME:
+ case TIMESTAMP: {
+ long val = buffer.getLong();
+ if (val < Long.MIN_VALUE + 1) {
+ tuple.put(i, DatumFactory.createNullDatum());
+ } else {
+ tuple.put(i, DatumFactory.createFromInt8(columnTypes[i], val));
+ }
+ break;
+ }
case NULL_TYPE:
tuple.put(i, NullDatum.get());
break;
@@ -442,10 +456,6 @@ public class RawFile {
buffer.put(strBytes2);
break;
- case TIMESTAMP:
- buffer.putLong(((TimestampDatum)t.get(i)).getMillis());
- break;
-
case BLOB : {
byte [] rawBytes = t.get(i).asByteArray();
if (flushBufferAndReplace(recordOffset, rawBytes.length + 4)) {
@@ -480,7 +490,13 @@ public class RawFile {
case INET4 :
buffer.put(t.get(i).asByteArray());
break;
-
+ case DATE:
+ buffer.putInt(t.get(i).asInt4());
+ break;
+ case TIME:
+ case TIMESTAMP:
+ buffer.putLong(t.get(i).asInt8());
+ break;
default:
throw new IOException("Cannot support data type: " + columnTypes[i].getType());
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/567183fc/tajo-storage/src/main/java/org/apache/tajo/storage/RowStoreUtil.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/RowStoreUtil.java b/tajo-storage/src/main/java/org/apache/tajo/storage/RowStoreUtil.java
index 9f32028..e54fb57 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/RowStoreUtil.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/RowStoreUtil.java
@@ -22,7 +22,6 @@ import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.datum.DatumFactory;
-import org.apache.tajo.datum.TimestampDatum;
import org.apache.tajo.util.Bytes;
import java.nio.ByteBuffer;
@@ -182,7 +181,10 @@ public class RowStoreUtil {
bb.put(_string);
break;
case DATE: bb.putInt(tuple.get(i).asInt4()); break;
- case TIMESTAMP: bb.putLong(((TimestampDatum)tuple.get(i)).getMillis()); break;
+ case TIME:
+ case TIMESTAMP:
+ bb.putLong(tuple.get(i).asInt8());
+ break;
case BLOB:
byte [] bytes = tuple.get(i).asByteArray();
bb.putInt(bytes.length);
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/567183fc/tajo-storage/src/main/java/org/apache/tajo/storage/StorageUtil.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageUtil.java b/tajo-storage/src/main/java/org/apache/tajo/storage/StorageUtil.java
index 9627a5d..1f13946 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageUtil.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/StorageUtil.java
@@ -40,20 +40,39 @@ public class StorageUtil {
}
public static int getColByteSize(Column col) {
- switch(col.getDataType().getType()) {
- case BOOLEAN: return 1;
- case CHAR: return 1;
- case BIT: return 1;
- case INT2: return 2;
- case INT4: return 4;
- case INT8: return 8;
- case FLOAT4: return 4;
- case FLOAT8: return 8;
- case INET4: return 4;
- case INET6: return 32;
- case TEXT: return 256;
- case BLOB: return 256;
- default: return 0;
+ switch (col.getDataType().getType()) {
+ case BOOLEAN:
+ return 1;
+ case CHAR:
+ return 1;
+ case BIT:
+ return 1;
+ case INT2:
+ return 2;
+ case INT4:
+ return 4;
+ case INT8:
+ return 8;
+ case FLOAT4:
+ return 4;
+ case FLOAT8:
+ return 8;
+ case INET4:
+ return 4;
+ case INET6:
+ return 32;
+ case TEXT:
+ return 256;
+ case BLOB:
+ return 256;
+ case DATE:
+ return 4;
+ case TIME:
+ return 8;
+ case TIMESTAMP:
+ return 8;
+ default:
+ return 0;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/567183fc/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java b/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java
index 16b370c..1685675 100644
--- a/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java
+++ b/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java
@@ -245,6 +245,7 @@ public class TestStorages {
assertEquals(tuple.get(i), retrieved.get(i));
}
}
+ scanner.close();
}
@Test
@@ -308,6 +309,7 @@ public class TestStorages {
assertEquals(tuple.get(i), retrieved.get(i));
}
}
+ scanner.close();
}
@Test
@@ -371,5 +373,46 @@ public class TestStorages {
assertEquals(tuple.get(i), retrieved.get(i));
}
}
+ scanner.close();
+ }
+
+ @Test
+ public void testTime() throws IOException {
+ if (storeType == StoreType.CSV || storeType == StoreType.RAW) {
+ Schema schema = new Schema();
+ schema.addColumn("col1", Type.DATE);
+ schema.addColumn("col2", Type.TIME);
+ schema.addColumn("col3", Type.TIMESTAMP);
+
+ Options options = new Options();
+ TableMeta meta = CatalogUtil.newTableMeta(storeType, options);
+
+ Path tablePath = new Path(testDir, "testTime.data");
+ Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
+ appender.init();
+
+ Tuple tuple = new VTuple(3);
+ tuple.put(new Datum[]{
+ DatumFactory.createDate("1980-04-01"),
+ DatumFactory.createTime("12:34:56"),
+ DatumFactory.createTimeStamp((int) System.currentTimeMillis() / 1000)
+ });
+ appender.addTuple(tuple);
+ appender.flush();
+ appender.close();
+
+ FileStatus status = fs.getFileStatus(tablePath);
+ FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
+ Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, fragment);
+ scanner.init();
+
+ Tuple retrieved;
+ while ((retrieved = scanner.next()) != null) {
+ for (int i = 0; i < tuple.size(); i++) {
+ assertEquals(tuple.get(i), retrieved.get(i));
+ }
+ }
+ scanner.close();
+ }
}
}