You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ch...@apache.org on 2014/06/18 19:36:10 UTC
svn commit: r1603547 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/PigConfiguration.java
src/org/apache/pig/backend/hadoop/executionengine/tez/TezJob.java
Author: cheolsoo
Date: Wed Jun 18 17:36:10 2014
New Revision: 1603547
URL: http://svn.apache.org/r1603547
Log:
PIG-4006: Make the interval of DAGStatus report configurable (cheolsoo)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/PigConfiguration.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJob.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1603547&r1=1603546&r2=1603547&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Wed Jun 18 17:36:10 2014
@@ -76,6 +76,8 @@ PIG-2207: Support custom counters for ag
IMPROVEMENTS
+PIG-4006: Make the interval of DAGStatus report configurable (cheolsoo)
+
PIG-3999: Document PIG-3388 (lbendig via cheolsoo)
PIG-3954: Document use of user level jar cache (aniket486)
Modified: pig/trunk/src/org/apache/pig/PigConfiguration.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/PigConfiguration.java?rev=1603547&r1=1603546&r2=1603547&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/PigConfiguration.java (original)
+++ pig/trunk/src/org/apache/pig/PigConfiguration.java Wed Jun 18 17:36:10 2014
@@ -92,6 +92,11 @@ public class PigConfiguration {
public static final String TEZ_SESSION_REUSE = "pig.tez.session.reuse";
/**
+ * This key is used to configure the interval of dag status report in seconds.
+ */
+ public static final String TEZ_DAG_STATUS_REPORT_INTERVAL = "pig.tez.dag.status.report.interval";
+
+ /**
* Turns off use of combiners in MapReduce jobs produced by Pig.
*/
public static final String PROP_NO_COMBINER = "pig.exec.nocombiner";
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJob.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJob.java?rev=1603547&r1=1603546&r2=1603547&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJob.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJob.java Wed Jun 18 17:36:10 2014
@@ -22,6 +22,8 @@ import java.util.EnumSet;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
+import java.util.Timer;
+import java.util.TimerTask;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
@@ -64,6 +66,8 @@ public class TezJob extends ControlledJo
private TezCounters dagCounters;
// Vertex, CounterGroup, Counter, Value
private Map<String, Map<String, Map<String, Long>>> vertexCounters;
+ // Timer for DAG status reporter
+ private Timer timer;
public TezJob(TezConfiguration conf, DAG dag, Map<String, LocalResource> requestAMResources)
throws IOException {
@@ -131,6 +135,10 @@ public class TezJob extends ControlledJo
return;
}
+ timer = new Timer();
+ timer.schedule(new DAGStatusReporter(), 1000, conf.getLong(
+ PigConfiguration.TEZ_DAG_STATUS_REPORT_INTERVAL, 10) * 1000);
+
while (true) {
try {
dagStatus = dagClient.getDAGStatus(statusGetOpts);
@@ -140,7 +148,6 @@ public class TezJob extends ControlledJo
break;
}
- log.info("DAG Status: " + dagStatus);
setJobState(dagState2JobState(dagStatus.getState()));
if (dagStatus.isCompleted()) {
StringBuilder sb = new StringBuilder();
@@ -171,6 +178,14 @@ public class TezJob extends ControlledJo
// Do nothing
}
}
+
+ timer.cancel();
+ }
+
+ private class DAGStatusReporter extends TimerTask {
+ public void run() {
+ log.info("DAG Status: " + dagStatus);
+ }
}
private void collectVertexCounters() {