You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by ji...@apache.org on 2013/12/28 07:36:32 UTC
[35/50] [abbrv] git commit: TAJO-448: Timestamp should be based on
unixtime. (hyunsik)
TAJO-448: Timestamp should be based on unixtime. (hyunsik)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tajo/commit/fce88829
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tajo/tree/fce88829
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tajo/diff/fce88829
Branch: refs/heads/DAG-execplan
Commit: fce88829d58885197f67c477699b57c632e7246d
Parents: ecc749e
Author: Hyunsik Choi <hy...@apache.org>
Authored: Tue Dec 24 10:20:14 2013 +0900
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Tue Dec 24 10:31:17 2013 +0900
----------------------------------------------------------------------
CHANGES.txt | 2 +
.../org/apache/tajo/datum/DatumFactory.java | 13 ++++---
.../org/apache/tajo/datum/TimestampDatum.java | 39 +++++++-------------
.../java/org/apache/tajo/json/DatumAdapter.java | 6 ++-
.../apache/tajo/datum/TestTimestampDatum.java | 10 ++---
.../function/datetime/ToCharTimestamp.java | 10 +++--
.../engine/function/datetime/ToTimestamp.java | 6 +--
.../tajo/engine/eval/TestSQLExpression.java | 17 +++++----
.../engine/function/TestDateTimeFunctions.java | 18 +++++++--
.../java/org/apache/tajo/storage/RawFile.java | 5 ++-
.../org/apache/tajo/storage/RowStoreUtil.java | 3 +-
11 files changed, 69 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/fce88829/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 5593cd3..a730a7a 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -168,6 +168,8 @@ Release 0.8.0 - unreleased
BUG FIXES
+ TAJO-448: Timestamp should be based on unixtime. (hyunsik)
+
TAJO-450: Incorrect inet4datum comparison. (jinho)
TAJO-182: Comparison of primitive values including null value should
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/fce88829/tajo-common/src/main/java/org/apache/tajo/datum/DatumFactory.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/DatumFactory.java b/tajo-common/src/main/java/org/apache/tajo/datum/DatumFactory.java
index 080d20c..0c98084 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/DatumFactory.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/DatumFactory.java
@@ -23,6 +23,7 @@ import org.apache.tajo.common.TajoDataTypes.DataType;
import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.datum.exception.InvalidCastException;
import org.apache.tajo.util.Bytes;
+import org.joda.time.DateTime;
import org.joda.time.LocalDate;
import org.joda.time.LocalTime;
@@ -165,7 +166,7 @@ public class DatumFactory {
case INT8:
return new Int8Datum(val);
case TIMESTAMP:
- return new TimestampDatum(val);
+ return createTimeStampFromMillis(val);
default:
throw new UnsupportedOperationException("Cannot create " + type.getType().name() + " datum from INT8");
}
@@ -267,8 +268,12 @@ public class DatumFactory {
return new TimeDatum(LocalTime.parse(dateStr));
}
- public static TimestampDatum createTimeStamp(long instance) {
- return new TimestampDatum(instance);
+ public static TimestampDatum createTimeStamp(int unixTime) {
+ return new TimestampDatum(unixTime);
+ }
+
+ public static TimestampDatum createTimeStampFromMillis(long millis) {
+ return new TimestampDatum(new DateTime(millis));
}
public static TimestampDatum createTimeStamp(String timeStamp) {
@@ -301,8 +306,6 @@ public class DatumFactory {
public static TimestampDatum createTimestamp(Datum datum) {
switch (datum.type()) {
- case INT8:
- return new TimestampDatum(datum.asInt8());
case TEXT:
return new TimestampDatum(datum.asChars());
case TIMESTAMP:
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/fce88829/tajo-common/src/main/java/org/apache/tajo/datum/TimestampDatum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/TimestampDatum.java b/tajo-common/src/main/java/org/apache/tajo/datum/TimestampDatum.java
index 4c53caa..4799964 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/TimestampDatum.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/TimestampDatum.java
@@ -19,7 +19,6 @@
package org.apache.tajo.datum;
import org.apache.tajo.common.TajoDataTypes;
-import org.apache.tajo.datum.exception.InvalidCastException;
import org.apache.tajo.datum.exception.InvalidOperationException;
import org.apache.tajo.util.Bytes;
import org.joda.time.DateTime;
@@ -33,9 +32,9 @@ public class TimestampDatum extends Datum {
private static final DateTimeFormatter DEFAULT_FORMATTER = DateTimeFormat.forPattern(DEFAULT_FORMAT_STRING);
private DateTime dateTime;
- public TimestampDatum(long instant) {
+ public TimestampDatum(int timestamp) {
super(TajoDataTypes.Type.TIMESTAMP);
- dateTime = new DateTime(instant);
+ dateTime = new DateTime((long)timestamp * 1000);
}
public TimestampDatum(DateTime dateTime) {
@@ -43,7 +42,7 @@ public class TimestampDatum extends Datum {
this.dateTime = dateTime;
}
- public TimestampDatum(byte [] bytes) {
+ TimestampDatum(byte [] bytes) {
super(TajoDataTypes.Type.TIMESTAMP);
this.dateTime = new DateTime(Bytes.toLong(bytes));
}
@@ -53,6 +52,14 @@ public class TimestampDatum extends Datum {
this.dateTime = DateTime.parse(datetime, DEFAULT_FORMATTER);
}
+ public int getUnixTime() {
+ return (int) (dateTime.getMillis() / 1000);
+ }
+
+ public long getMillis() {
+ return dateTime.getMillis();
+ }
+
public DateTime getDateTime() {
return dateTime;
}
@@ -98,37 +105,17 @@ public class TimestampDatum extends Datum {
}
@Override
- public int asInt4() {
- throw new InvalidCastException();
- }
-
- @Override
- public long asInt8() {
- return dateTime.getMillis();
- }
-
- @Override
- public float asFloat4() {
- throw new InvalidCastException();
- }
-
- @Override
- public double asFloat8() {
- throw new InvalidCastException();
- }
-
- @Override
public String asChars() {
return dateTime.toString(DEFAULT_FORMATTER);
}
- public String toChars(String format) {
+ public String toChars(DateTimeFormatter format) {
return dateTime.toString(format);
}
@Override
public int size() {
- return 8;
+ return SIZE;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/fce88829/tajo-common/src/main/java/org/apache/tajo/json/DatumAdapter.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/json/DatumAdapter.java b/tajo-common/src/main/java/org/apache/tajo/json/DatumAdapter.java
index 278caec..daece65 100644
--- a/tajo-common/src/main/java/org/apache/tajo/json/DatumAdapter.java
+++ b/tajo-common/src/main/java/org/apache/tajo/json/DatumAdapter.java
@@ -38,7 +38,7 @@ public class DatumAdapter implements GsonSerDerAdapter<Datum> {
case TIME:
return new TimeDatum(jsonObject.get("value").getAsLong());
case TIMESTAMP:
- return new TimestampDatum(jsonObject.get("value").getAsLong());
+ return new TimestampDatum(jsonObject.get("value").getAsString());
default:
return context.deserialize(jsonObject.get("body"),
DatumFactory.getDatumClass(TajoDataTypes.Type.valueOf(typeName)));
@@ -54,9 +54,11 @@ public class DatumAdapter implements GsonSerDerAdapter<Datum> {
jsonObj.addProperty("value", src.asInt4());
break;
case TIME:
- case TIMESTAMP:
jsonObj.addProperty("value", src.asInt8());
break;
+ case TIMESTAMP:
+ jsonObj.addProperty("value", src.asChars());
+ break;
default:
jsonObj.add("body", context.serialize(src));
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/fce88829/tajo-common/src/test/java/org/apache/tajo/datum/TestTimestampDatum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/test/java/org/apache/tajo/datum/TestTimestampDatum.java b/tajo-common/src/test/java/org/apache/tajo/datum/TestTimestampDatum.java
index 6691400..0482857 100644
--- a/tajo-common/src/test/java/org/apache/tajo/datum/TestTimestampDatum.java
+++ b/tajo-common/src/test/java/org/apache/tajo/datum/TestTimestampDatum.java
@@ -29,11 +29,11 @@ import static org.junit.Assert.assertEquals;
public class TestTimestampDatum {
- private static long timestamp;
+ private static int timestamp;
@BeforeClass
public static void setUp() {
- timestamp = System.currentTimeMillis();
+ timestamp = (int) (System.currentTimeMillis() / 1000);
}
@Test
@@ -48,7 +48,7 @@ public class TestTimestampDatum {
d.asInt4();
}
- @Test
+ @Test(expected = InvalidCastException.class)
public final void testAsInt8() {
Datum d = DatumFactory.createTimeStamp(timestamp);
assertEquals(timestamp, d.asInt8());
@@ -62,7 +62,7 @@ public class TestTimestampDatum {
@Test(expected = InvalidCastException.class)
public final void testAsFloat8() {
- long instance = 1386577582;
+ int instance = 1386577582;
Datum d = DatumFactory.createTimeStamp(instance);
d.asFloat8();
}
@@ -78,7 +78,7 @@ public class TestTimestampDatum {
public final void testAsByteArray() {
TimestampDatum d = DatumFactory.createTimeStamp(timestamp);
TimestampDatum copy = new TimestampDatum(d.asByteArray());
- assertEquals(d.asInt8(), copy.asInt8());
+ assertEquals(d, copy);
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/fce88829/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/datetime/ToCharTimestamp.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/datetime/ToCharTimestamp.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/datetime/ToCharTimestamp.java
index 42f406e..a3fd1f3 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/datetime/ToCharTimestamp.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/datetime/ToCharTimestamp.java
@@ -22,6 +22,7 @@ import org.apache.tajo.catalog.Column;
import org.apache.tajo.datum.Datum;
import org.apache.tajo.datum.DatumFactory;
import org.apache.tajo.datum.NullDatum;
+import org.apache.tajo.datum.TimestampDatum;
import org.apache.tajo.engine.eval.FunctionEval;
import org.apache.tajo.engine.function.GeneralFunction;
import org.apache.tajo.storage.Tuple;
@@ -52,15 +53,16 @@ public class ToCharTimestamp extends GeneralFunction {
@Override
public Datum eval(Tuple params) {
- Datum valueDatum = params.get(0);
- Datum pattern = params.get(1);
- if(valueDatum instanceof NullDatum || params instanceof NullDatum) {
+ if(params.isNull(0) || params.isNull(1)) {
return NullDatum.get();
}
+ TimestampDatum valueDatum = (TimestampDatum) params.get(0);
+ Datum pattern = params.get(1);
+
if (formatter == null || !constantFormat) {
formatter = DateTimeFormat.forPattern(pattern.asChars());
}
- return DatumFactory.createText(formatter.print(valueDatum.asInt8()));
+ return DatumFactory.createText(valueDatum.toChars(formatter));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/fce88829/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/datetime/ToTimestamp.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/datetime/ToTimestamp.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/datetime/ToTimestamp.java
index f333d20..6779b85 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/datetime/ToTimestamp.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/datetime/ToTimestamp.java
@@ -25,11 +25,11 @@ import org.apache.tajo.datum.NullDatum;
import org.apache.tajo.engine.function.GeneralFunction;
import org.apache.tajo.storage.Tuple;
-import static org.apache.tajo.common.TajoDataTypes.Type.INT8;
+import static org.apache.tajo.common.TajoDataTypes.Type.INT4;
public class ToTimestamp extends GeneralFunction {
public ToTimestamp() {
- super(new Column[] {new Column("timestamp", INT8)});
+ super(new Column[] {new Column("timestamp", INT4)});
}
@@ -39,6 +39,6 @@ public class ToTimestamp extends GeneralFunction {
if (value instanceof NullDatum) {
return NullDatum.get();
}
- return DatumFactory.createTimeStamp(value.asInt8());
+ return DatumFactory.createTimeStamp(value.asInt4());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/fce88829/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/eval/TestSQLExpression.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/eval/TestSQLExpression.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/eval/TestSQLExpression.java
index 9af33ad..f5d9042 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/eval/TestSQLExpression.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/eval/TestSQLExpression.java
@@ -19,6 +19,7 @@
package org.apache.tajo.engine.eval;
import org.apache.tajo.catalog.Schema;
+import org.apache.tajo.datum.TimestampDatum;
import org.junit.Test;
import java.io.IOException;
@@ -62,8 +63,10 @@ public class TestSQLExpression extends ExprTestBase {
@Test
public void testCastWithNestedFunction() throws IOException {
- testSimpleEval("select to_char(to_timestamp(CAST(split_part('1386577650.123', '.', 1) as INT8)), " +
- "'yyyy-MM-dd HH:mm:ss');", new String[] {"1970-01-17 10:09:37"});
+ int timestamp = (int) (System.currentTimeMillis() / 1000);
+ TimestampDatum expected = new TimestampDatum(timestamp);
+ testSimpleEval(String.format("select to_timestamp(CAST(split_part('%d.999', '.', 1) as INT8));", timestamp),
+ new String[] {expected.asChars()});
}
@Test
@@ -80,17 +83,15 @@ public class TestSQLExpression extends ExprTestBase {
new String[]{"1980-04-01 01:50:01", "234.0"});
testSimpleEval("select '1980-04-01 01:50:01'::timestamp;", new String [] {"1980-04-01 01:50:01"});
- testSimpleEval("select '1980-04-01 01:50:01'::timestamp::bigint::timestamp", new String [] {"1980-04-01 01:50:01"});
- testSimpleEval("select cast (('1980-04-01 01:50:01'::timestamp)::bigint as timestamp)",
- new String [] {"1980-04-01 01:50:01"});
- testSimpleEval("select to_timestamp(cast ('1970-01-17 10:09:37'::timestamp as int8));",
- new String [] {"1970-01-17 10:09:37"});
+ testSimpleEval("select '1980-04-01 01:50:01'::timestamp::text", new String [] {"1980-04-01 01:50:01"});
+
+ testSimpleEval("select (cast ('99999'::int8 as text))::int4 + 1", new String [] {"100000"});
}
@Test
public void testBooleanLiteral() throws IOException {
testSimpleEval("select true", new String[] {"t"});
- testSimpleEval("select false", new String[] {"f"});
+ testSimpleEval("select false", new String[]{"f"});
Schema schema = new Schema();
schema.addColumn("col1", TEXT);
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/fce88829/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/function/TestDateTimeFunctions.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/function/TestDateTimeFunctions.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/function/TestDateTimeFunctions.java
index c4a65a8..8e2db6e 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/function/TestDateTimeFunctions.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/function/TestDateTimeFunctions.java
@@ -19,7 +19,9 @@
package org.apache.tajo.engine.function;
+import org.apache.tajo.datum.TimestampDatum;
import org.apache.tajo.engine.eval.ExprTestBase;
+import org.joda.time.DateTime;
import org.junit.Test;
import java.io.IOException;
@@ -28,13 +30,21 @@ public class TestDateTimeFunctions extends ExprTestBase {
@Test
public void testToTimestamp() throws IOException {
- testSimpleEval("select to_timestamp('1970-01-17 10:09:37'::timestamp::int8);", new String[]{"1970-01-17 10:09:37"});
- testSimpleEval("select to_timestamp(cast (1386577582 as int8)) < to_timestamp(cast (1386577583 as int8));",
- new String[]{"t"});
+ long expectedTimestamp = System.currentTimeMillis();
+ DateTime expectedDateTime = new DateTime(expectedTimestamp);
+
+ // (expectedTimestamp / 1000) means the translation from millis seconds to unix timestamp
+ String q1 = String.format("select to_timestamp(%d);", (expectedTimestamp / 1000));
+ testSimpleEval(q1, new String[]{expectedDateTime.toString(TimestampDatum.DEFAULT_FORMAT_STRING)});
}
@Test
public void testToChar() throws IOException {
- testSimpleEval("select to_char(to_timestamp(1386577582), 'yyyy-MM');", new String[]{"1970-01"});
+ long expectedTimestamp = System.currentTimeMillis();
+ DateTime expectedDateTime = new DateTime(expectedTimestamp);
+ String dateFormatStr = "yyyy-MM";
+ // (expectedTimestamp / 1000) means the translation from millis seconds to unix timestamp
+ String q = String.format("select to_char(to_timestamp(%d), 'yyyy-MM');", (expectedTimestamp / 1000));
+ testSimpleEval(q, new String[]{expectedDateTime.toString(dateFormatStr)});
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/fce88829/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RawFile.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RawFile.java b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RawFile.java
index 7cce7e5..db511dc 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RawFile.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RawFile.java
@@ -30,6 +30,7 @@ import org.apache.tajo.common.TajoDataTypes.DataType;
import org.apache.tajo.datum.DatumFactory;
import org.apache.tajo.datum.NullDatum;
import org.apache.tajo.datum.ProtobufDatumFactory;
+import org.apache.tajo.datum.TimestampDatum;
import org.apache.tajo.storage.fragment.FileFragment;
import org.apache.tajo.util.BitArray;
@@ -212,7 +213,7 @@ public class RawFile {
break;
case TIMESTAMP:
- tuple.put(i, DatumFactory.createTimeStamp(buffer.getLong()));
+ tuple.put(i, DatumFactory.createTimeStampFromMillis(buffer.getLong()));
break;
case BLOB : {
@@ -442,7 +443,7 @@ public class RawFile {
break;
case TIMESTAMP:
- buffer.putLong(t.get(i).asInt8());
+ buffer.putLong(((TimestampDatum)t.get(i)).getMillis());
break;
case BLOB : {
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/fce88829/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RowStoreUtil.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RowStoreUtil.java b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RowStoreUtil.java
index 3aa0fbf..e5b9a8f 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RowStoreUtil.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RowStoreUtil.java
@@ -22,6 +22,7 @@ import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.datum.DatumFactory;
+import org.apache.tajo.datum.TimestampDatum;
import org.apache.tajo.util.Bytes;
import java.nio.ByteBuffer;
@@ -180,7 +181,7 @@ public class RowStoreUtil {
bb.put(_string);
break;
case DATE: bb.putInt(tuple.get(i).asInt4()); break;
- case TIMESTAMP: bb.putLong(tuple.get(i).asInt8()); break;
+ case TIMESTAMP: bb.putLong(((TimestampDatum)tuple.get(i)).getMillis()); break;
case BLOB:
byte [] bytes = tuple.get(i).asByteArray();
bb.putInt(bytes.length);