You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by th...@apache.org on 2012/08/24 02:18:09 UTC
svn commit: r1376800 [1/6] - in /pig/trunk: ./ .eclipse.templates/ conf/
contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/
contrib/zebra/src/java/org/apache/hadoop/zebra/pig/
contrib/zebra/src/java/org/apache/hadoop/zebra/pig/compa...
Author: thejas
Date: Fri Aug 24 00:18:05 2012
New Revision: 1376800
URL: http://svn.apache.org/viewvc?rev=1376800&view=rev
Log:
PIG-1314: Add DateTime Support to Pig (zjshen via thejas)
Added:
pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/comparator/DateTimeExpr.java
pig/trunk/src/org/apache/pig/backend/hadoop/DateTimeWritable.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java.orig
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigDateTimeRawComparator.java
pig/trunk/src/org/apache/pig/builtin/AddDuration.java
pig/trunk/src/org/apache/pig/builtin/CurrentTime.java
pig/trunk/src/org/apache/pig/builtin/DaysBetween.java
pig/trunk/src/org/apache/pig/builtin/GetDay.java
pig/trunk/src/org/apache/pig/builtin/GetHour.java
pig/trunk/src/org/apache/pig/builtin/GetMilliSecond.java
pig/trunk/src/org/apache/pig/builtin/GetMinute.java
pig/trunk/src/org/apache/pig/builtin/GetMonth.java
pig/trunk/src/org/apache/pig/builtin/GetSecond.java
pig/trunk/src/org/apache/pig/builtin/GetWeek.java
pig/trunk/src/org/apache/pig/builtin/GetWeekYear.java
pig/trunk/src/org/apache/pig/builtin/GetYear.java
pig/trunk/src/org/apache/pig/builtin/HoursBetween.java
pig/trunk/src/org/apache/pig/builtin/MilliSecondsBetween.java
pig/trunk/src/org/apache/pig/builtin/MinutesBetween.java
pig/trunk/src/org/apache/pig/builtin/MonthsBetween.java
pig/trunk/src/org/apache/pig/builtin/SecondsBetween.java
pig/trunk/src/org/apache/pig/builtin/SubtractDuration.java
pig/trunk/src/org/apache/pig/builtin/ToDate.java
pig/trunk/src/org/apache/pig/builtin/ToDate2ARGS.java
pig/trunk/src/org/apache/pig/builtin/ToDate3ARGS.java
pig/trunk/src/org/apache/pig/builtin/ToDateISO.java
pig/trunk/src/org/apache/pig/builtin/ToMilliSeconds.java
pig/trunk/src/org/apache/pig/builtin/ToString.java
pig/trunk/src/org/apache/pig/builtin/ToUnixTime.java
pig/trunk/src/org/apache/pig/builtin/WeeksBetween.java
pig/trunk/src/org/apache/pig/builtin/YearsBetween.java
pig/trunk/src/org/apache/pig/impl/io/NullableDateTimeWritable.java
Modified:
pig/trunk/.eclipse.templates/.classpath
pig/trunk/CHANGES.txt
pig/trunk/conf/pig.properties
pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/DBStorage.java
pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/SequenceFileLoader.java
pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/SchemaConverter.java
pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/comparator/ExprUtils.java
pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/schema/ColumnType.java
pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/schema/SchemaParser.jjt
pig/trunk/src/org/apache/pig/LoadCaster.java
pig/trunk/src/org/apache/pig/PigWarning.java
pig/trunk/src/org/apache/pig/StoreCaster.java
pig/trunk/src/org/apache/pig/backend/hadoop/HDataType.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapBase.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapReduce.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/partitioners/WeightedRangePartitioner.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/PhysicalOperator.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/ComparisonOperator.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/ConstantExpression.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/EqualToExpr.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/ExpressionOperator.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/GTOrEqualToExpr.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/GreaterThanExpr.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/LTOrEqualToExpr.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/LessThanExpr.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/NotEqualToExpr.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POBinCond.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POCast.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POIsNull.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POMapLookUp.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POProject.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserComparisonFunc.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POForEach.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POLocalRearrange.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPartialAgg.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPreCombinerLocalRearrange.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POSort.java
pig/trunk/src/org/apache/pig/backend/hadoop/hbase/HBaseBinaryConverter.java
pig/trunk/src/org/apache/pig/backend/hadoop/hbase/HBaseStorage.java
pig/trunk/src/org/apache/pig/builtin/BinStorage.java
pig/trunk/src/org/apache/pig/builtin/TextLoader.java
pig/trunk/src/org/apache/pig/builtin/Utf8StorageConverter.java
pig/trunk/src/org/apache/pig/data/BinInterSedes.java
pig/trunk/src/org/apache/pig/data/DataReaderWriter.java
pig/trunk/src/org/apache/pig/data/DataType.java
pig/trunk/src/org/apache/pig/data/DefaultTuple.java
pig/trunk/src/org/apache/pig/data/SchemaTuple.java
pig/trunk/src/org/apache/pig/data/SchemaTupleClassGenerator.java
pig/trunk/src/org/apache/pig/data/SizeUtil.java
pig/trunk/src/org/apache/pig/data/TypeAwareTuple.java
pig/trunk/src/org/apache/pig/impl/logicalLayer/schema/Schema.java
pig/trunk/src/org/apache/pig/impl/logicalLayer/schema/SchemaUtil.java
pig/trunk/src/org/apache/pig/impl/util/CastUtils.java
pig/trunk/src/org/apache/pig/impl/util/NumValCarrier.java
pig/trunk/src/org/apache/pig/impl/util/StorageUtil.java
pig/trunk/src/org/apache/pig/newplan/logical/visitor/TypeCheckingExpVisitor.java
pig/trunk/src/org/apache/pig/parser/AliasMasker.g
pig/trunk/src/org/apache/pig/parser/AstPrinter.g
pig/trunk/src/org/apache/pig/parser/AstValidator.g
pig/trunk/src/org/apache/pig/parser/LogicalPlanGenerator.g
pig/trunk/src/org/apache/pig/parser/QueryLexer.g
pig/trunk/src/org/apache/pig/parser/QueryParser.g
pig/trunk/src/org/apache/pig/pen/AugmentBaseDataVisitor.java
pig/trunk/src/org/apache/pig/scripting/jruby/JrubyScriptEngine.java
pig/trunk/src/org/apache/pig/scripting/jruby/RubySchema.java
pig/trunk/test/e2e/pig/udfs/java/org/apache/pig/test/udf/storefunc/PigPerformanceLoader.java
pig/trunk/test/org/apache/pig/data/TestSchemaTuple.java
pig/trunk/test/org/apache/pig/test/TestAdd.java
pig/trunk/test/org/apache/pig/test/TestBuiltin.java
pig/trunk/test/org/apache/pig/test/TestConversions.java
pig/trunk/test/org/apache/pig/test/TestDivide.java
pig/trunk/test/org/apache/pig/test/TestEqualTo.java
pig/trunk/test/org/apache/pig/test/TestGTOrEqual.java
pig/trunk/test/org/apache/pig/test/TestGreaterThan.java
pig/trunk/test/org/apache/pig/test/TestLTOrEqual.java
pig/trunk/test/org/apache/pig/test/TestLessThan.java
pig/trunk/test/org/apache/pig/test/TestMod.java
pig/trunk/test/org/apache/pig/test/TestMultiply.java
pig/trunk/test/org/apache/pig/test/TestNotEqualTo.java
pig/trunk/test/org/apache/pig/test/TestNull.java
pig/trunk/test/org/apache/pig/test/TestOrderBy.java
pig/trunk/test/org/apache/pig/test/TestPOBinCond.java
pig/trunk/test/org/apache/pig/test/TestPOCast.java
pig/trunk/test/org/apache/pig/test/TestPackage.java
pig/trunk/test/org/apache/pig/test/TestPigTupleRawComparator.java
pig/trunk/test/org/apache/pig/test/TestResourceSchema.java
pig/trunk/test/org/apache/pig/test/TestSchema.java
pig/trunk/test/org/apache/pig/test/TestStore.java
pig/trunk/test/org/apache/pig/test/TestSubtract.java
pig/trunk/test/org/apache/pig/test/TestTextDataParser.java
pig/trunk/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java
pig/trunk/test/org/apache/pig/test/utils/GenRandomData.java
Modified: pig/trunk/.eclipse.templates/.classpath
URL: http://svn.apache.org/viewvc/pig/trunk/.eclipse.templates/.classpath?rev=1376800&r1=1376799&r2=1376800&view=diff
==============================================================================
--- pig/trunk/.eclipse.templates/.classpath (original)
+++ pig/trunk/.eclipse.templates/.classpath Fri Aug 24 00:18:05 2012
@@ -25,7 +25,7 @@
<classpathentry exported="true" kind="lib" path="build/ivy/lib/Pig/js-1.7R2.jar"/>
<classpathentry exported="true" kind="lib" path="build/ivy/lib/Pig/zookeeper-3.3.3.jar"/>
<classpathentry exported="true" kind="lib" path="build/ivy/lib/Pig/guava-r06.jar"/>
- <classpathentry exported="true" kind="lib" path="build/ivy/lib/Pig/jython-2.5.0.jar"/>
+ <classpathentry exported="true" kind="lib" path="build/ivy/lib/Pig/jython-2.5.2.jar"/>
<classpathentry exported="true" kind="lib" path="build/ivy/lib/Pig/hadoop-core-0.20.2.jar"/>
<classpathentry exported="true" kind="lib" path="build/ivy/lib/Pig/commons-logging-1.1.1.jar"/>
<classpathentry exported="true" kind="lib" path="build/ivy/lib/Pig/commons-cli-1.2.jar"/>
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1376800&r1=1376799&r2=1376800&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri Aug 24 00:18:05 2012
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
IMPROVEMENTS
+PIG-1314: Add DateTime Support to Pig (zjshen via thejas)
+
PIG-2785: NoClassDefFoundError after upgrading to pig 0.10.0 from 0.9.0 (matterhayes via sms)
PIG-2556: CSVExcelStorage load: quoted field with newline as first character sees newline as record end (tivv via dvryaboy)
Modified: pig/trunk/conf/pig.properties
URL: http://svn.apache.org/viewvc/pig/trunk/conf/pig.properties?rev=1376800&r1=1376799&r2=1376800&view=diff
==============================================================================
--- pig/trunk/conf/pig.properties (original)
+++ pig/trunk/conf/pig.properties Fri Aug 24 00:18:05 2012
@@ -17,6 +17,10 @@
#exectype local|mapreduce, mapreduce is default
#exectype=local
+#the default timezone: if it is not set, the default timezone for this host is used.
+#the correct timezone format is the UTC offset: e.g., +08:00.
+#pig.datetime.default.tz=
+
#pig.logfile=
#Do not spill temp files smaller than this size (bytes)
Modified: pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/DBStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/DBStorage.java?rev=1376800&r1=1376799&r2=1376800&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/DBStorage.java (original)
+++ pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/DBStorage.java Fri Aug 24 00:18:05 2012
@@ -17,6 +17,8 @@
*/
package org.apache.pig.piggybank.storage;
+import org.joda.time.DateTime;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.NullWritable;
@@ -115,6 +117,11 @@ public class DBStorage extends StoreFunc
sqlPos++;
break;
+ case DataType.DATETIME:
+ ps.setDate(sqlPos, ((DateTime) field).toDate());
+ sqlPos++;
+ break;
+
case DataType.BYTEARRAY:
byte[] b = ((DataByteArray) field).get();
ps.setBytes(sqlPos, b);
Modified: pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/SequenceFileLoader.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/SequenceFileLoader.java?rev=1376800&r1=1376799&r2=1376800&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/SequenceFileLoader.java (original)
+++ pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/SequenceFileLoader.java Fri Aug 24 00:18:05 2012
@@ -21,6 +21,8 @@ import java.io.IOException;
import java.lang.reflect.Type;
import java.util.ArrayList;
+import org.joda.time.DateTime;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.BooleanWritable;
@@ -40,6 +42,7 @@ import org.apache.hadoop.mapreduce.lib.i
import org.apache.hadoop.mapreduce.lib.input.SequenceFileRecordReader;
import org.apache.pig.FileInputLoadFunc;
import org.apache.pig.backend.BackendException;
+import org.apache.pig.backend.hadoop.DateTimeWritable;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit;
import org.apache.pig.data.DataByteArray;
import org.apache.pig.data.DataType;
@@ -96,6 +99,7 @@ public class SequenceFileLoader extends
else if (t == DoubleWritable.class) return DataType.DOUBLE;
else if (t == BooleanWritable.class) return DataType.BOOLEAN;
else if (t == ByteWritable.class) return DataType.BYTE;
+ else if (t == DateTimeWritable.class) return DataType.DATETIME;
// not doing maps or other complex types for now
else return DataType.ERROR;
}
@@ -110,6 +114,7 @@ public class SequenceFileLoader extends
case DataType.FLOAT: return ((FloatWritable) w).get();
case DataType.DOUBLE: return ((DoubleWritable) w).get();
case DataType.BYTE: return ((ByteWritable) w).get();
+ case DataType.DATETIME: return ((DateTimeWritable) w).get();
}
return null;
Modified: pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/SchemaConverter.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/SchemaConverter.java?rev=1376800&r1=1376799&r2=1376800&view=diff
==============================================================================
--- pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/SchemaConverter.java (original)
+++ pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/SchemaConverter.java Fri Aug 24 00:18:05 2012
@@ -49,6 +49,9 @@ class SchemaConverter {
case DataType.BOOLEAN:
ret = ColumnType.BOOL;
break;
+ case DataType.DATETIME:
+ ret = ColumnType.DATETIME;
+ break;
case DataType.BAG:
ret = ColumnType.COLLECTION;
break;
Added: pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/comparator/DateTimeExpr.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/comparator/DateTimeExpr.java?rev=1376800&view=auto
==============================================================================
--- pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/comparator/DateTimeExpr.java (added)
+++ pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/comparator/DateTimeExpr.java Fri Aug 24 00:18:05 2012
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.zebra.pig.comparator;
+
+import org.joda.time.DateTime;
+
+
+public final class DateTimeExpr extends FixedLengthPrimitive {
+ private static final int ONE_MINUTE = 60000;
+ public DateTimeExpr(int index) {
+ super(index, (Long.SIZE + Short.SIZE) / Byte.SIZE);
+ }
+
+ protected void convertValue(byte[] b, Object o) {
+ if (b.length != (Long.SIZE + Short.SIZE) / Byte.SIZE) {
+ throw new IllegalArgumentException("Incorrect buffer size");
+ }
+ long dt = ((DateTime) o).getMillis();
+ dt ^= 1L << (Long.SIZE - 1);
+ b[7] = (byte) dt;
+ b[6] = (byte) (dt >> 8);
+ b[5] = (byte) (dt >> 16);
+ b[4] = (byte) (dt >> 24);
+ b[3] = (byte) (dt >> 32);
+ b[2] = (byte) (dt >> 40);
+ b[1] = (byte) (dt >> 48);
+ b[0] = (byte) (dt >> 56);
+
+ short dtz = (short) (((DateTime) o).getZone().getOffset((DateTime) o) / ONE_MINUTE);
+ dtz ^= 1 << (Short.SIZE - 1);
+ b[9] = (byte) dtz;
+ b[8] = (byte) (dtz >> 8);
+ }
+
+ @Override
+ protected String getType() {
+ return "DateTime";
+ }
+}
Modified: pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/comparator/ExprUtils.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/comparator/ExprUtils.java?rev=1376800&r1=1376799&r2=1376800&view=diff
==============================================================================
--- pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/comparator/ExprUtils.java (original)
+++ pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/pig/comparator/ExprUtils.java Fri Aug 24 00:18:05 2012
@@ -109,6 +109,8 @@ public class ExprUtils {
return new IntExpr(index);
case DataType.LONG:
return new LongExpr(index);
+ case DataType.DATETIME:
+ return new DateTimeExpr(index);
default:
throw new RuntimeException("Not a prmitive PIG type");
}
Modified: pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/schema/ColumnType.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/schema/ColumnType.java?rev=1376800&r1=1376799&r2=1376800&view=diff
==============================================================================
--- pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/schema/ColumnType.java (original)
+++ pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/schema/ColumnType.java Fri Aug 24 00:18:05 2012
@@ -79,6 +79,14 @@ public enum ColumnType {
}
},
/**
+ * DateTime
+ */
+ DATETIME("datetime") {
+ public byte pigDataType() {
+ return DataType.DATETIME;
+ }
+ },
+ /**
* Collection
*/
COLLECTION("collection") {
Modified: pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/schema/SchemaParser.jjt
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/schema/SchemaParser.jjt?rev=1376800&r1=1376799&r2=1376800&view=diff
==============================================================================
--- pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/schema/SchemaParser.jjt (original)
+++ pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/schema/SchemaParser.jjt Fri Aug 24 00:18:05 2012
@@ -57,6 +57,7 @@ TOKEN : { <BOOL : "boolean"> }
TOKEN : { <LONG : "long"> }
TOKEN : { <FLOAT : "float"> }
TOKEN : { <DOUBLE : "double"> }
+TOKEN : { <DATETIME : "datetime"> }
TOKEN : { <STRING : "string"> }
TOKEN : { <BYTES : "bytes"> }
TOKEN : { <COLLECTION : "collection"> }
Modified: pig/trunk/src/org/apache/pig/LoadCaster.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/LoadCaster.java?rev=1376800&r1=1376799&r2=1376800&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/LoadCaster.java (original)
+++ pig/trunk/src/org/apache/pig/LoadCaster.java Fri Aug 24 00:18:05 2012
@@ -19,6 +19,8 @@ package org.apache.pig;
import java.io.IOException;
import java.util.Map;
+import org.joda.time.DateTime;
+
import org.apache.pig.ResourceSchema.ResourceFieldSchema;
import org.apache.pig.classification.InterfaceAudience;
import org.apache.pig.classification.InterfaceStability;
@@ -70,6 +72,14 @@ public interface LoadCaster {
public Double bytesToDouble(byte[] b) throws IOException;
/**
+ * Cast data from bytearray to datetime value.
+ * @param b bytearray to be cast.
+ * @return datetime value.
+ * @throws IOException if the value cannot be cast.
+ */
+ public DateTime bytesToDateTime(byte[] b) throws IOException;
+
+ /**
* Cast data from bytearray to integer value.
* @param b bytearray to be cast.
* @return Double value.
Modified: pig/trunk/src/org/apache/pig/PigWarning.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/PigWarning.java?rev=1376800&r1=1376799&r2=1376800&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/PigWarning.java (original)
+++ pig/trunk/src/org/apache/pig/PigWarning.java Fri Aug 24 00:18:05 2012
@@ -34,6 +34,7 @@ public enum PigWarning {
IMPLICIT_CAST_TO_INT,
IMPLICIT_CAST_TO_LONG,
IMPLICIT_CAST_TO_BOOLEAN,
+ IMPLICIT_CAST_TO_DATETIME,
IMPLICIT_CAST_TO_MAP,
IMPLICIT_CAST_TO_TUPLE,
TOO_LARGE_FOR_INT,
Modified: pig/trunk/src/org/apache/pig/StoreCaster.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/StoreCaster.java?rev=1376800&r1=1376799&r2=1376800&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/StoreCaster.java (original)
+++ pig/trunk/src/org/apache/pig/StoreCaster.java Fri Aug 24 00:18:05 2012
@@ -20,6 +20,8 @@ package org.apache.pig;
import java.io.IOException;
import java.util.Map;
+import org.joda.time.DateTime;
+
import org.apache.pig.classification.InterfaceAudience;
import org.apache.pig.classification.InterfaceStability;
import org.apache.pig.data.DataBag;
@@ -48,6 +50,8 @@ public interface StoreCaster extends Loa
public byte[] toBytes(Long l) throws IOException;
+ public byte[] toBytes(DateTime dt) throws IOException;
+
public byte[] toBytes(Map<String, Object> m) throws IOException;
public byte[] toBytes(Tuple t) throws IOException;
Added: pig/trunk/src/org/apache/pig/backend/hadoop/DateTimeWritable.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/DateTimeWritable.java?rev=1376800&view=auto
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/DateTimeWritable.java (added)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/DateTimeWritable.java Fri Aug 24 00:18:05 2012
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.backend.hadoop;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.io.WritableComparator;
+
+/**
+ * Writable for Double values.
+ */
+public class DateTimeWritable implements WritableComparable {
+
+ private static final int ONE_MINUTE = 60000;
+
+ private DateTime value = null;
+
+ public DateTimeWritable() {
+
+ }
+
+ public DateTimeWritable(DateTime dt) {
+ value = dt;
+ }
+
+ public void readFields(DataInput in) throws IOException {
+ value = new DateTime(in.readLong(), DateTimeZone.forOffsetMillis(in.readShort() * ONE_MINUTE));
+ }
+
+ public void write(DataOutput out) throws IOException {
+ out.writeLong(value.getMillis());
+ out.writeShort(value.getZone().getOffset(value) / ONE_MINUTE);
+ }
+
+ public void set(DateTime dt) {
+ value = dt;
+ }
+
+ public DateTime get() {
+ return value;
+ }
+
+ /**
+ * Returns true iff <code>o</code> is a DateTimeWritable with the same
+ * value.
+ */
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof DateTimeWritable)) {
+ return false;
+ }
+ DateTimeWritable other = (DateTimeWritable) o;
+ return this.value.equals(other.value);
+ }
+
+ @Override
+ public int hashCode() {
+ return value.hashCode();
+ }
+
+ public int compareTo(Object o) {
+ DateTimeWritable other = (DateTimeWritable) o;
+ return value.compareTo(other.value);
+ }
+
+ @Override
+ public String toString() {
+ return value.toString();
+ }
+
+ /**
+ * A Comparator optimized for DateTimeWritable.
+ */
+ public static class Comparator extends WritableComparator {
+ public Comparator() {
+ super(DateTimeWritable.class);
+ }
+
+ @Override
+ public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
+ DateTime thisValue = new DateTime(readLong(b1, s1));
+ DateTime thatValue = new DateTime(readLong(b2, s2));
+ return thisValue.compareTo(thatValue);
+ }
+ }
+
+ static { // register this comparator
+ WritableComparator.define(DateTimeWritable.class, new Comparator());
+ }
+}
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/HDataType.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/HDataType.java?rev=1376800&r1=1376799&r2=1376800&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/HDataType.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/HDataType.java Fri Aug 24 00:18:05 2012
@@ -19,6 +19,8 @@ package org.apache.pig.backend.hadoop;
import java.util.Map;
+import org.joda.time.DateTime;
+
import org.apache.pig.PigException;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.data.DataBag;
@@ -32,6 +34,7 @@ import org.apache.pig.impl.io.NullableDo
import org.apache.pig.impl.io.NullableFloatWritable;
import org.apache.pig.impl.io.NullableIntWritable;
import org.apache.pig.impl.io.NullableLongWritable;
+import org.apache.pig.impl.io.NullableDateTimeWritable;
import org.apache.pig.impl.io.NullableText;
import org.apache.pig.impl.io.NullableTuple;
import org.apache.pig.impl.io.PigNullableWritable;
@@ -48,6 +51,7 @@ public class HDataType {
static NullableDoubleWritable doubleWrit = new NullableDoubleWritable();
static NullableIntWritable intWrit = new NullableIntWritable();
static NullableLongWritable longWrit = new NullableLongWritable();
+ static NullableDateTimeWritable dtWrit = new NullableDateTimeWritable();
static NullableBag defDB = new NullableBag();
static NullableTuple defTup = new NullableTuple();
static Map<Byte, String> typeToName = null;
@@ -82,7 +86,10 @@ public class HDataType {
case DataType.LONG:
return new NullableLongWritable((Long)o);
-
+
+ case DataType.DATETIME:
+ return new NullableDateTimeWritable((DateTime)o);
+
case DataType.TUPLE:
return new NullableTuple((Tuple)o);
@@ -126,6 +133,10 @@ public class HDataType {
NullableLongWritable nLongWrit = new NullableLongWritable();
nLongWrit.setNull(true);
return nLongWrit;
+ case DataType.DATETIME:
+ NullableDateTimeWritable nDateTimeWrit = new NullableDateTimeWritable();
+ nDateTimeWrit.setNull(true);
+ return nDateTimeWrit;
case DataType.TUPLE:
NullableTuple ntuple = new NullableTuple();
ntuple.setNull(true);
@@ -178,6 +189,9 @@ public class HDataType {
case DataType.LONG:
wcKey = longWrit;
break;
+ case DataType.DATETIME:
+ wcKey = dtWrit;
+ break;
case DataType.TUPLE:
wcKey = defTup;
break;
@@ -212,6 +226,8 @@ public class HDataType {
return DataType.INTEGER;
else if (o instanceof NullableLongWritable)
return DataType.LONG;
+ else if (o instanceof NullableDateTimeWritable)
+ return DataType.DATETIME;
else if (o instanceof NullableBag)
return DataType.BAG;
else if (o instanceof NullableTuple)
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java?rev=1376800&r1=1376799&r2=1376800&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java Fri Aug 24 00:18:05 2012
@@ -84,6 +84,7 @@ import org.apache.pig.impl.io.NullableDo
import org.apache.pig.impl.io.NullableFloatWritable;
import org.apache.pig.impl.io.NullableIntWritable;
import org.apache.pig.impl.io.NullableLongWritable;
+import org.apache.pig.impl.io.NullableDateTimeWritable;
import org.apache.pig.impl.io.NullablePartitionWritable;
import org.apache.pig.impl.io.NullableText;
import org.apache.pig.impl.io.NullableTuple;
@@ -956,6 +957,12 @@ public class JobControlCompiler{
}
}
+ public static class PigDateTimeWritableComparator extends PigWritableComparator {
+ public PigDateTimeWritableComparator() {
+ super(NullableDateTimeWritable.class);
+ }
+ }
+
public static class PigCharArrayWritableComparator extends PigWritableComparator {
public PigCharArrayWritableComparator() {
super(NullableText.class);
@@ -1012,6 +1019,12 @@ public class JobControlCompiler{
}
}
+ public static class PigGroupingDateTimeWritableComparator extends WritableComparator {
+ public PigGroupingDateTimeWritableComparator() {
+ super(NullableDateTimeWritable.class, true);
+ }
+ }
+
public static class PigGroupingCharArrayWritableComparator extends WritableComparator {
public PigGroupingCharArrayWritableComparator() {
super(NullableText.class, true);
@@ -1087,6 +1100,10 @@ public class JobControlCompiler{
job.setSortComparatorClass(PigDoubleRawComparator.class);
break;
+ case DataType.DATETIME:
+ job.setSortComparatorClass(PigDateTimeRawComparator.class);
+ break;
+
case DataType.CHARARRAY:
job.setSortComparatorClass(PigTextRawComparator.class);
break;
@@ -1141,6 +1158,11 @@ public class JobControlCompiler{
job.setGroupingComparatorClass(PigGroupingDoubleWritableComparator.class);
break;
+ case DataType.DATETIME:
+ job.setSortComparatorClass(PigDateTimeWritableComparator.class);
+ job.setGroupingComparatorClass(PigGroupingDateTimeWritableComparator.class);
+ break;
+
case DataType.CHARARRAY:
job.setSortComparatorClass(PigCharArrayWritableComparator.class);
job.setGroupingComparatorClass(PigGroupingCharArrayWritableComparator.class);