You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by kn...@apache.org on 2024/05/14 20:40:55 UTC
svn commit: r1917724 - in /pig/trunk: CHANGES.txt src/org/apache/pig/backend/hadoop/executionengine/spark/converter/IteratorTransform.java src/org/apache/pig/backend/hadoop/executionengine/spark/converter/SkewedJoinConverter.java
Author: knoguchi
Date: Tue May 14 20:40:55 2024
New Revision: 1917724
URL: http://svn.apache.org/viewvc?rev=1917724&view=rev
Log:
PIG-5447: Pig-on-Spark TestSkewedJoin.testSkewedJoinOuter failing with NoSuchElementException (knoguchi)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/IteratorTransform.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/SkewedJoinConverter.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1917724&r1=1917723&r2=1917724&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue May 14 20:40:55 2024
@@ -24,6 +24,7 @@ INCOMPATIBLE CHANGES
IMPROVEMENTS
PIG-5438: Update SparkCounter.Accumulator to AccumulatorV2 (knoguchi)
+
PIG-5439: Support Spark 3 and drop SparkShim (knoguchi)
OPTIMIZATIONS
@@ -31,6 +32,7 @@ OPTIMIZATIONS
BUG FIXES
PIG-5416: Spark unit tests failing randomly with "java.lang.RuntimeException: Unexpected job execution status RUNNING" (knoguchi)
+PIG-5447: Pig-on-Spark TestSkewedJoin.testSkewedJoinOuter failing with NoSuchElementException (knoguchi)
Release 0.18.0 - Unreleased
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/IteratorTransform.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/IteratorTransform.java?rev=1917724&r1=1917723&r2=1917724&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/IteratorTransform.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/IteratorTransform.java Tue May 14 20:40:55 2024
@@ -20,7 +20,7 @@ package org.apache.pig.backend.hadoop.ex
import java.util.Iterator;
abstract class IteratorTransform<IN, OUT> implements Iterator<OUT> {
- private Iterator<IN> delegate;
+ protected Iterator<IN> delegate;
public IteratorTransform(Iterator<IN> delegate) {
super();
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/SkewedJoinConverter.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/SkewedJoinConverter.java?rev=1917724&r1=1917723&r2=1917724&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/SkewedJoinConverter.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/SkewedJoinConverter.java Tue May 14 20:40:55 2024
@@ -172,7 +172,7 @@ public class SkewedJoinConverter impleme
}
public Iterator<Tuple> iterator() {
- return new IteratorTransform<Tuple2<PartitionIndexedKey, Tuple2<L, R>>, Tuple>(
+ return new CachedIteratorTransform<Tuple2<PartitionIndexedKey, Tuple2<L, R>>, Tuple>(
in) {
@Override
protected Tuple transform(