You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by kn...@apache.org on 2024/10/10 20:38:47 UTC
svn commit: r1921241 - in /pig/trunk: CHANGES.txt src/org/apache/pig/tools/pigstats/spark/SparkJobStats.java
Author: knoguchi
Date: Thu Oct 10 20:38:47 2024
New Revision: 1921241
URL: http://svn.apache.org/viewvc?rev=1921241&view=rev
Log:
PIG-5449: TestEmptyInputDir failing on pig-on-spark3 (knoguchi)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/tools/pigstats/spark/SparkJobStats.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1921241&r1=1921240&r2=1921241&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu Oct 10 20:38:47 2024
@@ -36,6 +36,8 @@ PIG-5457: Upgrade Zookeeper to 3.7.2 (fr
OPTIMIZATIONS
BUG FIXES
+PIG-5449: TestEmptyInputDir failing on pig-on-spark3 (knoguchi)
+
PIG-5459: Jython_Checkin_3 e2e failing with NoClassDefFoundError (hadoop3) (knoguchi)
PIG-5416: Spark unit tests failing randomly with "java.lang.RuntimeException: Unexpected job execution status RUNNING" (knoguchi)
Modified: pig/trunk/src/org/apache/pig/tools/pigstats/spark/SparkJobStats.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/pigstats/spark/SparkJobStats.java?rev=1921241&r1=1921240&r2=1921241&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/pigstats/spark/SparkJobStats.java (original)
+++ pig/trunk/src/org/apache/pig/tools/pigstats/spark/SparkJobStats.java Thu Oct 10 20:38:47 2024
@@ -21,6 +21,8 @@ package org.apache.pig.tools.pigstats.sp
import java.util.List;
import java.util.Map;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.Counters;
import org.apache.pig.PigWarning;
@@ -42,6 +44,7 @@ import org.apache.spark.executor.TaskMet
import com.google.common.collect.Maps;
public class SparkJobStats extends JobStats {
+ private static final Log LOG = LogFactory.getLog(SparkJobStats.class);
private int jobId;
private Map<String, Long> stats = Maps.newLinkedHashMap();
@@ -106,9 +109,10 @@ public class SparkJobStats extends JobSt
if (jobStatisticCollector != null) {
Map<String, List<TaskMetrics>> taskMetrics = jobStatisticCollector.getJobMetric(jobId);
if (taskMetrics == null) {
- throw new RuntimeException("No task metrics available for jobId " + jobId);
+ LOG.warn("No task metrics available for jobId " + jobId);
+ } else {
+ stats = combineTaskMetrics(taskMetrics);
}
- stats = combineTaskMetrics(taskMetrics);
}
}