You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by hy...@apache.org on 2013/12/11 09:14:39 UTC
git commit: TAJO-60: Implement Date Datum Type. (hyunsik)
Updated Branches:
refs/heads/master 93bae895f -> 2a3173757
TAJO-60: Implement Date Datum Type. (hyunsik)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tajo/commit/2a317375
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tajo/tree/2a317375
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tajo/diff/2a317375
Branch: refs/heads/master
Commit: 2a3173757cd74c91f625853e888314be33b36f56
Parents: 93bae89
Author: Hyunsik Choi <hy...@apache.org>
Authored: Wed Dec 11 17:14:12 2013 +0900
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Wed Dec 11 17:14:12 2013 +0900
----------------------------------------------------------------------
CHANGES.txt | 2 +
.../tajo/catalog/statistics/ColumnStats.java | 5 +-
.../tajo/catalog/statistics/TupleUtil.java | 63 --------
.../java/org/apache/tajo/datum/DateDatum.java | 152 +++++++++++++++++++
.../org/apache/tajo/datum/DatumFactory.java | 89 ++++++++---
.../org/apache/tajo/datum/TimestampDatum.java | 16 +-
.../org/apache/tajo/json/CommonGsonHelper.java | 4 +-
.../org/apache/tajo/json/DateDatumAdapter.java | 41 +++++
.../apache/tajo/json/TimestampDatumAdapter.java | 2 +-
.../org/apache/tajo/datum/TestDateDatum.java | 111 ++++++++++++++
.../apache/tajo/datum/TestTimestampDatum.java | 61 +++++---
.../tajo/master/event/TaskFatalErrorEvent.java | 12 +-
.../apache/tajo/master/querymaster/Query.java | 1 -
.../master/querymaster/QueryUnitAttempt.java | 25 ++-
.../org/apache/tajo/storage/RowStoreUtil.java | 32 ++--
.../tajo/storage/TextSerializeDeserialize.java | 5 +
16 files changed, 479 insertions(+), 142 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/2a317375/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 1e662ef..00e6b3d 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -4,6 +4,8 @@ Release 0.8.0 - unreleased
NEW FEATURES
+ TAJO-60: Implement Date Datum Type. (hyunsik)
+
TAJO-400: Implement pow(float8, float8) function. (DaeMyung Kang via jaehwa)
TAJO-384: to_bin(). (Jae Young Lee via jaehwa)
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/2a317375/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/ColumnStats.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/ColumnStats.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/ColumnStats.java
index 017c688..f9f6aab 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/ColumnStats.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/ColumnStats.java
@@ -25,6 +25,7 @@ import com.google.common.base.Objects;
import com.google.gson.annotations.Expose;
import com.google.protobuf.ByteString;
import org.apache.tajo.catalog.proto.CatalogProtos;
+import org.apache.tajo.datum.DatumFactory;
import org.apache.tajo.json.GsonObject;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.json.CatalogGsonHelper;
@@ -57,10 +58,10 @@ public class ColumnStats implements ProtoObject<CatalogProtos.ColumnStatsProto>,
this.numNulls = proto.getNumNulls();
}
if (proto.hasMinValue()) {
- this.minValue = TupleUtil.createFromBytes(getColumn().getDataType(), proto.getMinValue().toByteArray());
+ this.minValue = DatumFactory.createFromBytes(getColumn().getDataType(), proto.getMinValue().toByteArray());
}
if (proto.hasMaxValue()) {
- this.maxValue = TupleUtil.createFromBytes(getColumn().getDataType(), proto.getMaxValue().toByteArray());
+ this.maxValue = DatumFactory.createFromBytes(getColumn().getDataType(), proto.getMaxValue().toByteArray());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/2a317375/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/TupleUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/TupleUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/TupleUtil.java
deleted file mode 100644
index 556b324..0000000
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/TupleUtil.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.catalog.statistics;
-
-import com.google.protobuf.Message;
-import org.apache.tajo.common.TajoDataTypes.DataType;
-import org.apache.tajo.datum.*;
-
-import java.io.IOException;
-
-public class TupleUtil {
- public static Datum createFromBytes(DataType type, byte [] bytes) {
- switch (type.getType()) {
- case BOOLEAN:
- return new BooleanDatum(bytes);
- case BLOB:
- return new BlobDatum(bytes);
- case CHAR:
- return new CharDatum(bytes);
- case INT2:
- return new Int2Datum(bytes);
- case INT4:
- return new Int4Datum(bytes);
- case INT8:
- return new Int8Datum(bytes);
- case FLOAT4:
- return new Float4Datum(bytes);
- case FLOAT8:
- return new Float8Datum(bytes);
- case TEXT:
- return new TextDatum(bytes);
- case INET4:
- return new Inet4Datum(bytes);
- case PROTOBUF:
- ProtobufDatumFactory factory = ProtobufDatumFactory.get(type);
- Message.Builder builder = factory.newBuilder();
- try {
- builder.mergeFrom(bytes);
- return factory.createDatum(builder.build());
- } catch (IOException e) {
- e.printStackTrace();
- throw new RuntimeException(e);
- }
- default: throw new UnsupportedOperationException(type + " is not supported yet");
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/2a317375/tajo-common/src/main/java/org/apache/tajo/datum/DateDatum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/DateDatum.java b/tajo-common/src/main/java/org/apache/tajo/datum/DateDatum.java
new file mode 100644
index 0000000..4dac59e
--- /dev/null
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/DateDatum.java
@@ -0,0 +1,152 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.datum;
+
+import org.apache.tajo.common.TajoDataTypes;
+import org.apache.tajo.datum.exception.InvalidCastException;
+import org.apache.tajo.datum.exception.InvalidOperationException;
+import org.apache.tajo.util.Bytes;
+import org.joda.time.LocalDate;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+
+public class DateDatum extends Datum {
+ public static final int SIZE = 4;
+ /** ISO 8601/SQL standard format - ex) 1997-12-17 */
+ public static final String DEFAULT_FORMAT_STRING = "yyyy-MM-dd";
+ private static final DateTimeFormatter DEFAULT_FORMATTER = DateTimeFormat.forPattern(DEFAULT_FORMAT_STRING);
+ private LocalDate date;
+
+ public DateDatum(int value) {
+ super(TajoDataTypes.Type.DATE);
+ date = decode(value);
+ }
+
+ public DateDatum(LocalDate date) {
+ super(TajoDataTypes.Type.DATE);
+ this.date = date;
+ }
+
+ public DateDatum(byte [] bytes) {
+ this(Bytes.toInt(bytes));
+ }
+
+ public int getYear() {
+ return date.getYear();
+ }
+
+ public int getMonthOfYear() {
+ return date.getMonthOfYear();
+ }
+
+ public int getDayOfWeek() {
+ return date.getDayOfWeek();
+ }
+
+ public int getDayOfMonth() {
+ return date.getDayOfMonth();
+ }
+
+ public String toString() {
+ return asChars();
+ }
+
+ @Override
+ public int asInt4() {
+ return encode();
+ }
+
+ private static LocalDate decode(int val) {
+ int year = (val >> 16);
+ int monthOfYear = (0x0FFF & val) >> 8;
+ int dayOfMonth = (0xF0FF & val);
+ return new LocalDate(year, monthOfYear, dayOfMonth);
+ }
+
+ /**
+ * Year MonthOfYear DayOfMonth
+ * 31-16 15-8 7 - 0
+ *
+ * 0xFF 0xFF 0xFF 0xFF
+ */
+ private int encode() {
+ int instance = 0;
+ instance |= (date.getYear() << 16); // 1970 ~ : 2 bytes
+ instance |= (date.getMonthOfYear() << 8); // 1 - 12 : 1 byte
+ instance |= (date.getDayOfMonth()); // 0 - 31 : 1 byte
+ return instance;
+ }
+
+ @Override
+ public long asInt8() {
+ return encode();
+ }
+
+ @Override
+ public float asFloat4() {
+ throw new InvalidCastException();
+ }
+
+ @Override
+ public double asFloat8() {
+ throw new InvalidCastException();
+ }
+
+ @Override
+ public String asChars() {
+ return date.toString(DEFAULT_FORMATTER);
+ }
+
+ public String toChars(String format) {
+ return date.toString(format);
+ }
+
+ @Override
+ public int size() {
+ return SIZE;
+ }
+
+ @Override
+ public byte [] asByteArray() {
+ return Bytes.toBytes(encode());
+ }
+
+ @Override
+ public int compareTo(Datum datum) {
+ if (datum.type() == TajoDataTypes.Type.DATE) {
+ return date.compareTo(((DateDatum)datum).date);
+ } else {
+ throw new InvalidOperationException();
+ }
+ }
+
+ public boolean equals(Object obj) {
+ if (obj instanceof DateDatum) {
+ DateDatum another = (DateDatum) obj;
+ return date.isEqual(another.date);
+ } else {
+ throw new InvalidOperationException();
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return date.hashCode();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/2a317375/tajo-common/src/main/java/org/apache/tajo/datum/DatumFactory.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/DatumFactory.java b/tajo-common/src/main/java/org/apache/tajo/datum/DatumFactory.java
index 6fd9c35..d25c7d8 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/DatumFactory.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/DatumFactory.java
@@ -18,16 +18,15 @@
package org.apache.tajo.datum;
+import com.google.protobuf.Message;
import org.apache.tajo.common.TajoDataTypes.DataType;
import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.util.Bytes;
-import org.joda.time.DateTime;
+import org.joda.time.LocalDate;
-public class DatumFactory {
+import java.io.IOException;
- public static Datum create(DataType type, byte[] val) {
- return create(type.getType(), val);
- }
+public class DatumFactory {
public static Class<? extends Datum> getDatumClass(Type type) {
switch (type) {
@@ -47,6 +46,10 @@ public class DatumFactory {
return CharDatum.class;
case TEXT:
return TextDatum.class;
+ case TIMESTAMP:
+ return TimestampDatum.class;
+ case DATE:
+ return DateDatum.class;
case BIT:
return BitDatum.class;
case BLOB:
@@ -62,33 +65,69 @@ public class DatumFactory {
}
}
- public static Datum create(Type type, byte[] val) {
- switch (type) {
+ public static Datum createFromBytes(DataType dataType, byte[] bytes) {
+ switch (dataType.getType()) {
case BOOLEAN:
- return createBool(val[0]);
+ return createBool(bytes[0]);
case INT2:
- return createInt2(Bytes.toShort(val));
+ return createInt2(Bytes.toShort(bytes));
case INT4:
- return createInt4(Bytes.toInt(val));
+ return createInt4(Bytes.toInt(bytes));
case INT8:
- return createInt8(Bytes.toLong(val));
+ return createInt8(Bytes.toLong(bytes));
case FLOAT4:
- return createFloat4(Bytes.toFloat(val));
+ return createFloat4(Bytes.toFloat(bytes));
case FLOAT8:
- return createFloat8(Bytes.toDouble(val));
+ return createFloat8(Bytes.toDouble(bytes));
case CHAR:
- return createChar(val);
+ return createChar(bytes);
case TEXT:
- return createText(val);
+ return createText(bytes);
+ case DATE:
+ return new DateDatum(bytes);
+ case TIMESTAMP:
+ return new TimestampDatum(bytes);
case BIT:
- return createBit(val[0]);
+ return createBit(bytes[0]);
case BLOB:
- return createBlob(val);
+ return createBlob(bytes);
case INET4:
- return createInet4(val);
+ return createInet4(bytes);
+ case PROTOBUF:
+ ProtobufDatumFactory factory = ProtobufDatumFactory.get(dataType);
+ Message.Builder builder = factory.newBuilder();
+ try {
+ builder.mergeFrom(bytes);
+ return factory.createDatum(builder.build());
+ } catch (IOException e) {
+ e.printStackTrace();
+ throw new RuntimeException(e);
+ }
default:
- throw new UnsupportedOperationException(type.toString());
+ throw new UnsupportedOperationException(dataType.toString());
+ }
+ }
+
+ public static Datum createFromInt4(DataType type, int val) {
+ switch (type.getType()) {
+ case INT4:
+ return new Int4Datum(val);
+ case DATE:
+ return new DateDatum(val);
+ default:
+ throw new UnsupportedOperationException("Cannot create " + type.getType().name() + " datum from INT4");
+ }
+ }
+
+ public static Datum createFromInt8(DataType type, long val) {
+ switch (type.getType()) {
+ case INT8:
+ return new Int8Datum(val);
+ case DATE:
+ return new TimestampDatum(val);
+ default:
+ throw new UnsupportedOperationException("Cannot create " + type.getType().name() + " datum from INT8");
}
}
@@ -169,12 +208,20 @@ public class DatumFactory {
return new TextDatum(val);
}
+ public static DateDatum createDate(int instance) {
+ return new DateDatum(instance);
+ }
+
+ public static DateDatum createDate(String dateStr) {
+ return new DateDatum(LocalDate.parse(dateStr));
+ }
+
public static TimestampDatum createTimeStamp(long instance) {
return new TimestampDatum(instance);
}
- public static TimestampDatum createTimeStamp(String val) {
- return new TimestampDatum(DateTime.parse(val));
+ public static TimestampDatum createTimeStamp(String timeStamp) {
+ return new TimestampDatum(timeStamp);
}
public static TextDatum createText(byte[] val) {
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/2a317375/tajo-common/src/main/java/org/apache/tajo/datum/TimestampDatum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/TimestampDatum.java b/tajo-common/src/main/java/org/apache/tajo/datum/TimestampDatum.java
index c9bbd92..7f40e5e 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/TimestampDatum.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/TimestampDatum.java
@@ -23,11 +23,14 @@ import org.apache.tajo.datum.exception.InvalidCastException;
import org.apache.tajo.datum.exception.InvalidOperationException;
import org.apache.tajo.util.Bytes;
import org.joda.time.DateTime;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
public class TimestampDatum extends Datum {
public static final int SIZE = 8;
/** ISO 8601/SQL standard format - ex) 1997-12-17 07:37:16-08 */
- public static final String DEFAULT_FORMAT = "yyyy-MM-dd HH:mm:ss";
+ public static final String DEFAULT_FORMAT_STRING = "yyyy-MM-dd HH:mm:ss";
+ private static final DateTimeFormatter DEFAULT_FORMATTER = DateTimeFormat.forPattern(DEFAULT_FORMAT_STRING);
private DateTime dateTime;
public TimestampDatum(long instant) {
@@ -45,6 +48,11 @@ public class TimestampDatum extends Datum {
this.dateTime = new DateTime(Bytes.toLong(bytes));
}
+ public TimestampDatum(String datetime) {
+ super(TajoDataTypes.Type.TIMESTAMP);
+ this.dateTime = DateTime.parse(datetime, DEFAULT_FORMATTER);
+ }
+
public int getYear() {
return dateTime.getYear();
}
@@ -65,6 +73,10 @@ public class TimestampDatum extends Datum {
return dateTime.getHourOfDay();
}
+ public int getMinuteOfHour() {
+ return dateTime.getMinuteOfHour();
+ }
+
public int getSecondOfDay() {
return dateTime.getSecondOfDay();
}
@@ -103,7 +115,7 @@ public class TimestampDatum extends Datum {
@Override
public String asChars() {
- return dateTime.toString(DEFAULT_FORMAT);
+ return dateTime.toString(DEFAULT_FORMATTER);
}
public String toChars(String format) {
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/2a317375/tajo-common/src/main/java/org/apache/tajo/json/CommonGsonHelper.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/json/CommonGsonHelper.java b/tajo-common/src/main/java/org/apache/tajo/json/CommonGsonHelper.java
index b6d6779..fb32959 100644
--- a/tajo-common/src/main/java/org/apache/tajo/json/CommonGsonHelper.java
+++ b/tajo-common/src/main/java/org/apache/tajo/json/CommonGsonHelper.java
@@ -20,6 +20,7 @@ package org.apache.tajo.json;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
+import org.apache.tajo.datum.DateDatum;
import org.apache.tajo.datum.Datum;
import org.apache.tajo.datum.TimestampDatum;
import org.apache.tajo.util.TUtil;
@@ -36,8 +37,9 @@ public class CommonGsonHelper {
private static Map<Type, GsonSerDerAdapter> registerAdapters() {
Map<Type, GsonSerDerAdapter> adapters = TUtil.newHashMap();
- adapters.put(TimestampDatum.class, new TimestampDatumAdapter());
adapters.put(Datum.class, new DatumAdapter());
+ adapters.put(TimestampDatum.class, new TimestampDatumAdapter());
+ adapters.put(DateDatum.class, new DateDatumAdapter());
return adapters;
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/2a317375/tajo-common/src/main/java/org/apache/tajo/json/DateDatumAdapter.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/json/DateDatumAdapter.java b/tajo-common/src/main/java/org/apache/tajo/json/DateDatumAdapter.java
new file mode 100644
index 0000000..1a7c1a5
--- /dev/null
+++ b/tajo-common/src/main/java/org/apache/tajo/json/DateDatumAdapter.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.json;
+
+import com.google.gson.*;
+import org.apache.tajo.datum.DateDatum;
+
+import java.lang.reflect.Type;
+
+public class DateDatumAdapter implements GsonSerDerAdapter<DateDatum> {
+
+ @Override
+ public DateDatum deserialize(JsonElement json, Type typeOfT,
+ JsonDeserializationContext context) throws JsonParseException {
+ JsonObject jsonObject = json.getAsJsonObject();
+ return new DateDatum(jsonObject.get("val").getAsInt());
+ }
+
+ @Override
+ public JsonElement serialize(DateDatum src, Type typeOfSrc, JsonSerializationContext context) {
+ JsonObject jsonObj = new JsonObject();
+ jsonObj.addProperty("val", src.asInt4());
+ return jsonObj;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/2a317375/tajo-common/src/main/java/org/apache/tajo/json/TimestampDatumAdapter.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/json/TimestampDatumAdapter.java b/tajo-common/src/main/java/org/apache/tajo/json/TimestampDatumAdapter.java
index f963eb2..6b362a4 100644
--- a/tajo-common/src/main/java/org/apache/tajo/json/TimestampDatumAdapter.java
+++ b/tajo-common/src/main/java/org/apache/tajo/json/TimestampDatumAdapter.java
@@ -35,7 +35,7 @@ public class TimestampDatumAdapter implements GsonSerDerAdapter<TimestampDatum>
@Override
public JsonElement serialize(TimestampDatum src, Type typeOfSrc, JsonSerializationContext context) {
JsonObject jsonObj = new JsonObject();
- jsonObj.addProperty("value", src.asInt8());
+ jsonObj.addProperty("val", src.asInt8());
return jsonObj;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/2a317375/tajo-common/src/test/java/org/apache/tajo/datum/TestDateDatum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/test/java/org/apache/tajo/datum/TestDateDatum.java b/tajo-common/src/test/java/org/apache/tajo/datum/TestDateDatum.java
new file mode 100644
index 0000000..8d3b146
--- /dev/null
+++ b/tajo-common/src/test/java/org/apache/tajo/datum/TestDateDatum.java
@@ -0,0 +1,111 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.datum;
+
+import org.apache.tajo.common.TajoDataTypes.Type;
+import org.apache.tajo.datum.exception.InvalidCastException;
+import org.apache.tajo.json.CommonGsonHelper;
+import org.junit.Test;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+public class TestDateDatum {
+ private static String DATE = "1980-04-01";
+
+ @Test
+ public final void testType() {
+ Datum d = DatumFactory.createDate(DATE);
+ assertEquals(Type.DATE, d.type());
+ }
+
+ @Test
+ public final void testAsInt4() {
+ Datum d = DatumFactory.createDate(DATE);
+ Datum copy = DatumFactory.createDate(d.asInt4());
+ assertEquals(d, copy);
+ }
+
+ @Test
+ public final void testAsInt8() {
+ Datum d = DatumFactory.createDate(DATE);
+ Datum copy = DatumFactory.createDate((int) d.asInt8());
+ assertEquals(d, copy);
+ }
+
+ @Test(expected = InvalidCastException.class)
+ public final void testAsFloat4() {
+ Datum d = DatumFactory.createDate(DATE);
+ d.asFloat4();
+ }
+
+ @Test(expected = InvalidCastException.class)
+ public final void testAsFloat8() {
+ Datum d = DatumFactory.createDate(DATE);
+ d.asFloat8();
+ }
+
+ @Test
+ public final void testAsText() {
+ Datum d = DatumFactory.createDate(DATE);
+ Datum copy = DatumFactory.createDate(d.asChars());
+ assertEquals(d, copy);
+ }
+
+ @Test
+ public final void testAsByteArray() {
+ DateDatum d = DatumFactory.createDate(DATE);
+ DateDatum copy = new DateDatum(d.asByteArray());
+ assertEquals(d.asInt8(), copy.asInt8());
+ }
+
+ @Test
+ public final void testSize() {
+ Datum d = DatumFactory.createDate(DATE);
+ assertEquals(DateDatum.SIZE, d.asByteArray().length);
+ }
+
+ @Test
+ public final void testAsTextBytes() {
+ Datum d = DatumFactory.createDate(DATE);
+ assertArrayEquals(d.toString().getBytes(), d.asTextBytes());
+ }
+
+ @Test
+ public final void testToJson() {
+ Datum d = DatumFactory.createDate(DATE);
+ Datum copy = CommonGsonHelper.fromJson(d.toJson(), Datum.class);
+ assertEquals(d, copy);
+ }
+
+ @Test
+ public final void testInstance() {
+ DateDatum d = DatumFactory.createDate(DATE);
+ DateDatum copy = new DateDatum(d.asInt4());
+ assertEquals(d, copy);
+ }
+
+ @Test
+ public final void testGetFields() {
+ DateDatum d = DatumFactory.createDate(DATE);
+ assertEquals(1980, d.getYear());
+ assertEquals(4, d.getMonthOfYear());
+ assertEquals(1, d.getDayOfMonth());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/2a317375/tajo-common/src/test/java/org/apache/tajo/datum/TestTimestampDatum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/test/java/org/apache/tajo/datum/TestTimestampDatum.java b/tajo-common/src/test/java/org/apache/tajo/datum/TestTimestampDatum.java
index 7a62d06..6691400 100644
--- a/tajo-common/src/test/java/org/apache/tajo/datum/TestTimestampDatum.java
+++ b/tajo-common/src/test/java/org/apache/tajo/datum/TestTimestampDatum.java
@@ -20,38 +20,43 @@ package org.apache.tajo.datum;
import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.datum.exception.InvalidCastException;
+import org.apache.tajo.json.CommonGsonHelper;
+import org.junit.BeforeClass;
import org.junit.Test;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
public class TestTimestampDatum {
-
+
+ private static long timestamp;
+
+ @BeforeClass
+ public static void setUp() {
+ timestamp = System.currentTimeMillis();
+ }
+
@Test
public final void testType() {
- long instance = 1386577582;
- Datum d = DatumFactory.createTimeStamp(instance);
+ Datum d = DatumFactory.createTimeStamp(timestamp);
assertEquals(Type.TIMESTAMP, d.type());
}
@Test(expected = InvalidCastException.class)
public final void testAsInt4() {
- long instance = 1386577582;
- Datum d = DatumFactory.createTimeStamp(instance);
+ Datum d = DatumFactory.createTimeStamp(timestamp);
d.asInt4();
}
@Test
public final void testAsInt8() {
- long instance = 1386577582;
- Datum d = DatumFactory.createTimeStamp(instance);
- assertEquals(instance, d.asInt8());
+ Datum d = DatumFactory.createTimeStamp(timestamp);
+ assertEquals(timestamp, d.asInt8());
}
@Test(expected = InvalidCastException.class)
public final void testAsFloat4() {
- long instance = 1386577582;
- Datum d = DatumFactory.createTimeStamp(instance);
+ Datum d = DatumFactory.createTimeStamp(timestamp);
d.asFloat4();
}
@@ -64,37 +69,45 @@ public class TestTimestampDatum {
@Test
public final void testAsText() {
- long instance = 1386577582;
- Datum d = DatumFactory.createTimeStamp(instance);
- System.out.println(d.asChars());
+ Datum d = DatumFactory.createTimeStamp("1980-04-01 01:50:01");
+ Datum copy = DatumFactory.createTimeStamp(d.asChars());
+ assertEquals(d, copy);
}
@Test
public final void testAsByteArray() {
- long instance = 1386577582;
- TimestampDatum d = DatumFactory.createTimeStamp(instance);
+ TimestampDatum d = DatumFactory.createTimeStamp(timestamp);
TimestampDatum copy = new TimestampDatum(d.asByteArray());
assertEquals(d.asInt8(), copy.asInt8());
}
-
+
@Test
public final void testSize() {
- long instance = 1386577582;
- Datum d = DatumFactory.createTimeStamp(instance);
- assertEquals(TimestampDatum.SIZE, d.size());
+ Datum d = DatumFactory.createTimeStamp(timestamp);
+ assertEquals(TimestampDatum.SIZE, d.asByteArray().length);
}
@Test
public final void testAsTextBytes() {
- long instance = 1386577582;
- Datum d = DatumFactory.createTimeStamp(instance);
+ Datum d = DatumFactory.createTimeStamp("1980-04-01 01:50:01");
assertArrayEquals(d.toString().getBytes(), d.asTextBytes());
}
@Test
public final void testToJson() {
- long instance = 1386577582;
- Datum d = DatumFactory.createTimeStamp(instance);
- System.out.println(d.toJson());
+ Datum d = DatumFactory.createTimeStamp(timestamp);
+ Datum copy = CommonGsonHelper.fromJson(d.toJson(), Datum.class);
+ assertEquals(d, copy);
+ }
+
+ @Test
+ public final void testGetFields() {
+ TimestampDatum d = DatumFactory.createTimeStamp("1980-04-01 01:50:01");
+ assertEquals(1980, d.getYear());
+ assertEquals(4, d.getMonthOfYear());
+ assertEquals(1, d.getDayOfMonth());
+ assertEquals(1, d.getHourOfDay());
+ assertEquals(50, d.getMinuteOfHour());
+ assertEquals(01, d.getSecondOfMinute());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/2a317375/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/event/TaskFatalErrorEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/event/TaskFatalErrorEvent.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/event/TaskFatalErrorEvent.java
index d70de8a..a4d9900 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/event/TaskFatalErrorEvent.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/event/TaskFatalErrorEvent.java
@@ -22,14 +22,20 @@ import org.apache.tajo.QueryUnitAttemptId;
import org.apache.tajo.ipc.TajoWorkerProtocol.TaskFatalErrorReport;
public class TaskFatalErrorEvent extends TaskAttemptEvent {
- private TaskFatalErrorReport report;
+ private final String message;
+
public TaskFatalErrorEvent(TaskFatalErrorReport report) {
super(new QueryUnitAttemptId(report.getId()),
TaskAttemptEventType.TA_FATAL_ERROR);
- this.report = report;
+ this.message = report.getErrorMessage();
+ }
+
+ public TaskFatalErrorEvent(QueryUnitAttemptId attemptId, String message) {
+ super(attemptId, TaskAttemptEventType.TA_FATAL_ERROR);
+ this.message = message;
}
public String errorMessage() {
- return report.getErrorMessage();
+ return message;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/2a317375/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java
index c17b726..245c897 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java
@@ -69,7 +69,6 @@ public class Query implements EventHandler<QueryEvent> {
private final QueryId id;
private long appSubmitTime;
private long startTime;
- private long initializationTime;
private long finishTime;
private TableDesc resultDesc;
private int completedSubQueryCount = 0;
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/2a317375/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryUnitAttempt.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryUnitAttempt.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryUnitAttempt.java
index accb549..1af7332 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryUnitAttempt.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryUnitAttempt.java
@@ -268,31 +268,28 @@ public class QueryUnitAttempt implements EventHandler<TaskAttemptEvent> {
}
}
- private static class SucceededTransition
- implements SingleArcTransition<QueryUnitAttempt, TaskAttemptEvent>{
+ private static class SucceededTransition implements SingleArcTransition<QueryUnitAttempt, TaskAttemptEvent> {
@Override
public void transition(QueryUnitAttempt taskAttempt,
TaskAttemptEvent event) {
TaskCompletionReport report = ((TaskCompletionEvent)event).getReport();
- taskAttempt.fillTaskStatistics(report);
- taskAttempt.eventHandler.handle(new TaskTAttemptEvent(taskAttempt.getId(),
- TaskEventType.T_ATTEMPT_SUCCEEDED));
+ try {
+ taskAttempt.fillTaskStatistics(report);
+ taskAttempt.eventHandler.handle(new TaskTAttemptEvent(taskAttempt.getId(), TaskEventType.T_ATTEMPT_SUCCEEDED));
+ } catch (Throwable t) {
+ taskAttempt.eventHandler.handle(new TaskFatalErrorEvent(taskAttempt.getId(), t.getMessage()));
+ }
}
}
- private static class FailedTransition
- implements SingleArcTransition<QueryUnitAttempt, TaskAttemptEvent>{
+ private static class FailedTransition implements SingleArcTransition<QueryUnitAttempt, TaskAttemptEvent>{
@Override
- public void transition(QueryUnitAttempt taskAttempt,
- TaskAttemptEvent event) {
+ public void transition(QueryUnitAttempt taskAttempt, TaskAttemptEvent event) {
TaskFatalErrorEvent errorEvent = (TaskFatalErrorEvent) event;
- taskAttempt.eventHandler.handle(
- new TaskTAttemptEvent(taskAttempt.getId(),
- TaskEventType.T_ATTEMPT_FAILED));
- LOG.error("FROM " + taskAttempt.getHost() + " >> "
- + errorEvent.errorMessage());
+ taskAttempt.eventHandler.handle(new TaskTAttemptEvent(taskAttempt.getId(), TaskEventType.T_ATTEMPT_FAILED));
taskAttempt.addDiagnosticInfo(errorEvent.errorMessage());
+ LOG.error("FROM " + taskAttempt.getHost() + " >> " + errorEvent.errorMessage());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/2a317375/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RowStoreUtil.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RowStoreUtil.java b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RowStoreUtil.java
index c53dd4f..3aa0fbf 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RowStoreUtil.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RowStoreUtil.java
@@ -20,6 +20,7 @@ package org.apache.tajo.storage;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.Schema;
+import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.datum.DatumFactory;
import org.apache.tajo.util.Bytes;
@@ -51,10 +52,11 @@ public class RowStoreUtil {
ByteBuffer bb = ByteBuffer.wrap(bytes);
Tuple tuple = new VTuple(schema.getColumnNum());
Column col;
+ TajoDataTypes.DataType type;
for (int i =0; i < schema.getColumnNum(); i++) {
col = schema.getColumn(i);
-
- switch (col.getDataType().getType()) {
+ type = col.getDataType();
+ switch (type.getType()) {
case BOOLEAN: tuple.put(i, DatumFactory.createBool(bb.get())); break;
case BIT:
byte b = bb.get();
@@ -84,18 +86,22 @@ public class RowStoreUtil {
break;
case INT4:
+ case DATE:
int i_ = bb.getInt();
if ( i_ < Integer.MIN_VALUE + 1) {
tuple.put(i, DatumFactory.createNullDatum());
- }else {
- tuple.put(i, DatumFactory.createInt4(i_));
- }break;
+ } else {
+ tuple.put(i, DatumFactory.createFromInt4(type, i_));
+ }
+ break;
+
case INT8:
+ case TIMESTAMP:
long l = bb.getLong();
if ( l < Long.MIN_VALUE + 1) {
tuple.put(i, DatumFactory.createNullDatum());
}else {
- tuple.put(i, DatumFactory.createInt8(l));
+ tuple.put(i, DatumFactory.createFromInt8(type, l));
}
break;
@@ -114,7 +120,9 @@ public class RowStoreUtil {
tuple.put(i, DatumFactory.createNullDatum());
}else {
tuple.put(i, DatumFactory.createFloat8(d));
- }break;
+ }
+ break;
+
case TEXT:
byte [] _string = new byte[bb.getInt()];
bb.get(_string);
@@ -131,9 +139,11 @@ public class RowStoreUtil {
bb.get(_bytes);
if(Bytes.compareTo(bytes, Bytes.toBytes("NULL")) == 0) {
tuple.put(i, DatumFactory.createNullDatum());
- }else {
- tuple.put(i, DatumFactory.createBlob(_bytes));
- }break;
+ } else {
+ tuple.put(i, DatumFactory.createBlob(_bytes));
+ }
+ break;
+
case INET4:
byte [] _ipv4 = new byte[4];
bb.get(_ipv4);
@@ -169,6 +179,8 @@ public class RowStoreUtil {
bb.putInt(_string.length);
bb.put(_string);
break;
+ case DATE: bb.putInt(tuple.get(i).asInt4()); break;
+ case TIMESTAMP: bb.putLong(tuple.get(i).asInt8()); break;
case BLOB:
byte [] bytes = tuple.get(i).asByteArray();
bb.putInt(bytes.length);
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/2a317375/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/TextSerializeDeserialize.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/TextSerializeDeserialize.java b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/TextSerializeDeserialize.java
index 4280783..6afd9b9 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/TextSerializeDeserialize.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/TextSerializeDeserialize.java
@@ -76,6 +76,7 @@ public class TextSerializeDeserialize implements SerializeDeserialize {
case FLOAT4:
case FLOAT8:
case INET4:
+ case DATE:
case TIMESTAMP:
bytes = datum.asTextBytes();
length = bytes.length;
@@ -144,6 +145,10 @@ public class TextSerializeDeserialize implements SerializeDeserialize {
: DatumFactory.createText(chars);
break;
}
+ case DATE:
+ datum = isNull(bytes, offset, length, nullCharacters) ? NullDatum.get()
+ : DatumFactory.createDate(new String(bytes, offset, length));
+ break;
case TIMESTAMP:
datum = isNull(bytes, offset, length, nullCharacters) ? NullDatum.get()
: DatumFactory.createTimeStamp(new String(bytes, offset, length));