You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ro...@apache.org on 2014/11/20 20:33:53 UTC
svn commit: r1640792 - in /pig/trunk: ./ src/org/apache/pig/
src/org/apache/pig/backend/hadoop/executionengine/fetch/
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/
src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/ src/o...
Author: rohini
Date: Thu Nov 20 19:33:52 2014
New Revision: 1640792
URL: http://svn.apache.org/r1640792
Log:
PIG-4334: PigProcessor does not set pig.datetime.default.tz (rohini)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/PigConfiguration.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/fetch/FetchLauncher.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapBase.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapReduce.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PigProcessor.java
pig/trunk/src/org/apache/pig/impl/util/Utils.java
pig/trunk/src/org/apache/pig/tools/pigstats/tez/TezDAGStats.java
pig/trunk/test/org/apache/pig/test/TestDefaultDateTimeZone.java
pig/trunk/test/tez-local-tests
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1640792&r1=1640791&r2=1640792&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu Nov 20 19:33:52 2014
@@ -43,7 +43,17 @@ PIG-4247: S3 properties are not picked u
PIG-4242: For indented xmls with multiline content (e.g. wikipedia) XMLLoader cuts out the begining of every line
(holdfenytolvaj via daijy)
-Release 0.14.0 - Unreleased
+Release 0.14.1 - Unreleased
+
+INCOMPATIBLE CHANGES
+
+IMPROVEMENTS
+
+BUG FIXES
+
+PIG-4334: PigProcessor does not set pig.datetime.default.tz (rohini)
+
+Release 0.14.0
INCOMPATIBLE CHANGES
Modified: pig/trunk/src/org/apache/pig/PigConfiguration.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/PigConfiguration.java?rev=1640792&r1=1640791&r2=1640792&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/PigConfiguration.java (original)
+++ pig/trunk/src/org/apache/pig/PigConfiguration.java Thu Nov 20 19:33:52 2014
@@ -290,6 +290,11 @@ public class PigConfiguration {
*/
public static final String PIG_NO_TASK_REPORT = "pig.stats.notaskreport";
+ /**
+ * The timezone to be used by Pig datetime datatype
+ */
+ public static final String PIG_DATETIME_DEFAULT_TIMEZONE = "pig.datetime.default.tz";
+
// Pig on Tez runtime settings
/**
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/fetch/FetchLauncher.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/fetch/FetchLauncher.java?rev=1640792&r1=1640791&r2=1640792&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/fetch/FetchLauncher.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/fetch/FetchLauncher.java Thu Nov 20 19:33:52 2014
@@ -44,10 +44,10 @@ import org.apache.pig.impl.plan.Dependen
import org.apache.pig.impl.plan.PlanException;
import org.apache.pig.impl.plan.VisitorException;
import org.apache.pig.impl.util.UDFContext;
+import org.apache.pig.impl.util.Utils;
import org.apache.pig.tools.pigstats.EmptyPigStats;
import org.apache.pig.tools.pigstats.PigStats;
import org.apache.pig.tools.pigstats.PigStatusReporter;
-import org.joda.time.DateTimeZone;
/**
* This class is responsible for executing the fetch task, saving the result to disk
@@ -141,11 +141,7 @@ public class FetchLauncher {
udfContext.serialize(conf);
PigMapReduce.sJobConfInternal.set(conf);
- String dtzStr = conf.get("pig.datetime.default.tz");
- if (dtzStr != null && dtzStr.length() > 0) {
- // don't use offsets because it breaks across DST/Standard Time
- DateTimeZone.setDefault(DateTimeZone.forID(dtzStr));
- }
+ Utils.setDefaultTimeZone(conf);
boolean aggregateWarning = "true".equalsIgnoreCase(conf.get("aggregate.warning"));
PigStatusReporter pigStatusReporter = PigStatusReporter.getInstance();
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapBase.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapBase.java?rev=1640792&r1=1640791&r2=1640792&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapBase.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapBase.java Thu Nov 20 19:33:52 2014
@@ -22,7 +22,6 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import org.joda.time.DateTimeZone;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -54,6 +53,7 @@ import org.apache.pig.impl.plan.VisitorE
import org.apache.pig.impl.util.ObjectSerializer;
import org.apache.pig.impl.util.Pair;
import org.apache.pig.impl.util.SpillableMemoryManager;
+import org.apache.pig.impl.util.Utils;
import org.apache.pig.tools.pigstats.PigStatusReporter;
/**
@@ -215,11 +215,7 @@ public abstract class PigGenericMapBase
log.info("Aliases being processed per job phase (AliasName[line,offset]): " + job.get("pig.alias.location"));
- String dtzStr = PigMapReduce.sJobConfInternal.get().get("pig.datetime.default.tz");
- if (dtzStr != null && dtzStr.length() > 0) {
- // don't use offsets because it breaks across DST/Standard Time
- DateTimeZone.setDefault(DateTimeZone.forID(dtzStr));
- }
+ Utils.setDefaultTimeZone(PigMapReduce.sJobConfInternal.get());
}
/**
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapReduce.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapReduce.java?rev=1640792&r1=1640791&r2=1640792&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapReduce.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapReduce.java Thu Nov 20 19:33:52 2014
@@ -60,8 +60,8 @@ import org.apache.pig.impl.util.ObjectSe
import org.apache.pig.impl.util.Pair;
import org.apache.pig.impl.util.SpillableMemoryManager;
import org.apache.pig.impl.util.UDFContext;
+import org.apache.pig.impl.util.Utils;
import org.apache.pig.tools.pigstats.PigStatusReporter;
-import org.joda.time.DateTimeZone;
/**
* This class is the static Mapper & Reducer classes that
@@ -362,11 +362,7 @@ public class PigGenericMapReduce {
log.info("Aliases being processed per job phase (AliasName[line,offset]): " + jConf.get("pig.alias.location"));
- String dtzStr = PigMapReduce.sJobConfInternal.get().get("pig.datetime.default.tz");
- if (dtzStr != null && dtzStr.length() > 0) {
- // don't use offsets because it breaks across DST/Standard Time
- DateTimeZone.setDefault(DateTimeZone.forID(dtzStr));
- }
+ Utils.setDefaultTimeZone(PigMapReduce.sJobConfInternal.get());
}
/**
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PigProcessor.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PigProcessor.java?rev=1640792&r1=1640791&r2=1640792&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PigProcessor.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PigProcessor.java Thu Nov 20 19:33:52 2014
@@ -53,6 +53,7 @@ import org.apache.pig.impl.plan.Dependen
import org.apache.pig.impl.plan.VisitorException;
import org.apache.pig.impl.util.ObjectSerializer;
import org.apache.pig.impl.util.UDFContext;
+import org.apache.pig.impl.util.Utils;
import org.apache.pig.tools.pigstats.PigStatusReporter;
import org.apache.tez.common.TezUtils;
import org.apache.tez.dag.api.UserPayload;
@@ -135,6 +136,8 @@ public class PigProcessor extends Abstra
// for backwards compatibility with the existing code base.
PigMapReduce.sJobConfInternal.set(conf);
+ Utils.setDefaultTimeZone(conf);
+
boolean aggregateWarning = "true".equalsIgnoreCase(pc.getProperties().getProperty("aggregate.warning"));
PigStatusReporter pigStatusReporter = PigStatusReporter.getInstance();
pigStatusReporter.setContext(new TezTaskContext(getContext()));
Modified: pig/trunk/src/org/apache/pig/impl/util/Utils.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/impl/util/Utils.java?rev=1640792&r1=1640791&r2=1640792&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/impl/util/Utils.java (original)
+++ pig/trunk/src/org/apache/pig/impl/util/Utils.java Thu Nov 20 19:33:52 2014
@@ -71,6 +71,7 @@ import org.apache.pig.impl.logicalLayer.
import org.apache.pig.newplan.logical.relational.LogicalSchema;
import org.apache.pig.parser.ParserException;
import org.apache.pig.parser.QueryParserDriver;
+import org.joda.time.DateTimeZone;
import com.google.common.collect.Lists;
import com.google.common.primitives.Longs;
@@ -681,4 +682,12 @@ public class Utils {
}
return ret;
}
+
+ public static void setDefaultTimeZone(Configuration conf) {
+ String dtzStr = conf.get(PigConfiguration.PIG_DATETIME_DEFAULT_TIMEZONE);
+ if (dtzStr != null && dtzStr.length() > 0) {
+ // don't use offsets because it breaks across DST/Standard Time
+ DateTimeZone.setDefault(DateTimeZone.forID(dtzStr));
+ }
+ }
}
Modified: pig/trunk/src/org/apache/pig/tools/pigstats/tez/TezDAGStats.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/pigstats/tez/TezDAGStats.java?rev=1640792&r1=1640791&r2=1640792&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/pigstats/tez/TezDAGStats.java (original)
+++ pig/trunk/src/org/apache/pig/tools/pigstats/tez/TezDAGStats.java Thu Nov 20 19:33:52 2014
@@ -186,16 +186,18 @@ public class TezDAGStats extends JobStat
TezCounters tezCounters = tezJob.getDAGCounters();
if (tezCounters != null) {
counters = covertToHadoopCounters(tezCounters);
- }
- CounterGroup dagGrp = tezCounters.getGroup(DAG_COUNTER_GROUP);
- totalTasks = (int) dagGrp.findCounter("TOTAL_LAUNCHED_TASKS").getValue();
+ CounterGroup dagGrp = tezCounters.getGroup(DAG_COUNTER_GROUP);
+ totalTasks = (int) dagGrp.findCounter("TOTAL_LAUNCHED_TASKS").getValue();
- CounterGroup fsGrp = tezCounters.getGroup(FS_COUNTER_GROUP);
- fileBytesRead = fsGrp.findCounter("FILE_BYTES_READ").getValue();
- fileBytesWritten = fsGrp.findCounter("FILE_BYTES_WRITTEN").getValue();
- hdfsBytesRead = fsGrp.findCounter("HDFS_BYTES_READ").getValue();
- hdfsBytesWritten = fsGrp.findCounter("HDFS_BYTES_WRITTEN").getValue();
+ CounterGroup fsGrp = tezCounters.getGroup(FS_COUNTER_GROUP);
+ fileBytesRead = fsGrp.findCounter("FILE_BYTES_READ").getValue();
+ fileBytesWritten = fsGrp.findCounter("FILE_BYTES_WRITTEN").getValue();
+ hdfsBytesRead = fsGrp.findCounter("HDFS_BYTES_READ").getValue();
+ hdfsBytesWritten = fsGrp.findCounter("HDFS_BYTES_WRITTEN").getValue();
+ } else {
+ LOG.warn("Failed to get counters for DAG: " + dag.getName());
+ }
for (Entry<String, TezVertexStats> entry : tezVertexStatsMap.entrySet()) {
Vertex v = dag.getVertex(entry.getKey());
@@ -206,6 +208,10 @@ public class TezDAGStats extends JobStat
vertexStats.setConf(conf);
VertexStatus status = tezJob.getVertexStatus(v.getName());
+ if (status == null) {
+ LOG.warn("Failed to get status for vertex: " + v.getName());
+ continue;
+ }
vertexStats.accumulateStats(status, v.getParallelism());
if(vertexStats.getInputs() != null && !vertexStats.getInputs().isEmpty()) {
inputs.addAll(vertexStats.getInputs());
Modified: pig/trunk/test/org/apache/pig/test/TestDefaultDateTimeZone.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestDefaultDateTimeZone.java?rev=1640792&r1=1640791&r2=1640792&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestDefaultDateTimeZone.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestDefaultDateTimeZone.java Thu Nov 20 19:33:52 2014
@@ -29,7 +29,6 @@ import java.util.Properties;
import junit.framework.TestCase;
-import org.apache.pig.ExecType;
import org.apache.pig.PigServer;
import org.apache.pig.builtin.ToDate;
import org.apache.pig.data.Tuple;
@@ -44,6 +43,7 @@ public class TestDefaultDateTimeZone ext
private File tmpFile;
private DateTimeZone currentDTZ;
+ @Override
@Before
public void setUp() throws Exception {
currentDTZ = DateTimeZone.getDefault();
@@ -63,6 +63,7 @@ public class TestDefaultDateTimeZone ext
}
+ @Override
@After
public void tearDown() throws Exception {
tmpFile.delete();
@@ -75,7 +76,7 @@ public class TestDefaultDateTimeZone ext
.forOffsetMillis(DateTimeZone.forID("+08:00").getOffset(null)));
Properties config = new Properties();
config.setProperty("pig.datetime.default.tz", "+08:00");
- PigServer pig = new PigServer(ExecType.LOCAL, config);
+ PigServer pig = new PigServer(Util.getLocalTestMode(), config);
pig.registerQuery("a = load '"
+ Util.encodeEscape(Util.generateURI(tmpFile.toString(), pig.getPigContext()))
+ "' as (test:datetime);");
@@ -94,20 +95,20 @@ public class TestDefaultDateTimeZone ext
String defaultDTZ = "America/New_York"; // a timezone that uses DST
Properties config = new Properties();
config.setProperty("pig.datetime.default.tz", defaultDTZ);
- PigServer pig = new PigServer(ExecType.LOCAL, config);
+ PigServer pig = new PigServer(Util.getLocalTestMode(), config);
pig.registerQuery("a = load '"
+ Util.encodeEscape(Util.generateURI(tmpFile.toString(), pig.getPigContext()))
+ "' as (test:datetime);");
pig.registerQuery("b = filter a by test > ToDate('2014-01-01T00:00:00.000');");
pig.registerQuery("c = foreach b generate ToString(test, 'Z') as tz;");
Iterator<Tuple> actualItr = pig.openIterator("c");
-
+
Tuple est = actualItr.next();
assertEquals(Util.buildTuple("-0500"), est);
Tuple edt = actualItr.next();
assertEquals(Util.buildTuple("-0400"), edt);
}
-
+
private static Iterator<Tuple> generateExpectedResults(DateTimeZone dtz)
throws Exception {
List<Tuple> expectedResults = new ArrayList<Tuple>();
@@ -125,8 +126,7 @@ public class TestDefaultDateTimeZone ext
public void testTimeZone() throws IOException {
// Usually set through "pig.datetime.default.tz"
String defaultDTZ = "+03:00";
- DateTimeZone.setDefault(DateTimeZone.forOffsetMillis(DateTimeZone.forID(defaultDTZ)
- .getOffset(null)));
+ DateTimeZone.setDefault(DateTimeZone.forID(defaultDTZ));
String[] inputs = {
"1970-01-01T00:00:00.000-08:00",
"1970-01-01T00:00",
Modified: pig/trunk/test/tez-local-tests
URL: http://svn.apache.org/viewvc/pig/trunk/test/tez-local-tests?rev=1640792&r1=1640791&r2=1640792&view=diff
==============================================================================
--- pig/trunk/test/tez-local-tests (original)
+++ pig/trunk/test/tez-local-tests Thu Nov 20 19:33:52 2014
@@ -1,5 +1,6 @@
**/TestAccumuloPigCluster.java
**/TestBigTypeSort.java
**/TestCurrentTime.java
+**/TestDefaultDateTimeZone.java
**/TestHBaseStorage.java
**/TestInvokerGenerator.java