You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ro...@apache.org on 2013/12/06 01:44:57 UTC
svn commit: r1548342 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/impl/builtin/ReadScalars.java
Author: rohini
Date: Fri Dec 6 00:44:57 2013
New Revision: 1548342
URL: http://svn.apache.org/r1548342
Log:
PIG-2629: Wrong Usage of Scalar which is null causes high namenode operation (rohini)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/impl/builtin/ReadScalars.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1548342&r1=1548341&r2=1548342&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri Dec 6 00:44:57 2013
@@ -123,6 +123,8 @@ PIG-3480: TFile-based tmpfile compressio
BUG FIXES
+PIG-2629: Wrong Usage of Scalar which is null causes high namenode operation (rohini)
+
PIG-3593: Import jython standard module fail on cluster (daijy)
PIG-3576: NPE due to PIG-3549 when job never gets submitted (lbendig via cheolsoo)
Modified: pig/trunk/src/org/apache/pig/impl/builtin/ReadScalars.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/impl/builtin/ReadScalars.java?rev=1548342&r1=1548341&r2=1548342&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/impl/builtin/ReadScalars.java (original)
+++ pig/trunk/src/org/apache/pig/impl/builtin/ReadScalars.java Fri Dec 6 00:44:57 2013
@@ -23,12 +23,12 @@ import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.pig.EvalFunc;
import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.data.DataBag;
import org.apache.pig.data.DataType;
import org.apache.pig.data.Tuple;
import org.apache.pig.impl.io.InterStorage;
import org.apache.pig.impl.io.ReadToEndLoader;
import org.apache.pig.impl.util.UDFContext;
-import org.apache.pig.data.DataBag;
/**
* ReadScalars reads a line from a file and returns it as its value. The
@@ -40,22 +40,26 @@ public class ReadScalars extends EvalFun
private String scalarfilename = null;
// private String charset = "UTF-8";
private Object value = null;
-
+
// in-core input : used by illustrator
private Map<String, DataBag> inputBuffer = null;
+ private boolean valueLoaded = false;
+
/**
* Java level API
- *
+ *
* @param input
* expects a single constant that is the name of the file to be
* read
*/
@Override
public Object exec(Tuple input) throws IOException {
- if (value == null) {
- if (input == null || input.size() == 0)
+ if (!valueLoaded) {
+ if (input == null || input.size() == 0) {
+ valueLoaded = true;
return null;
+ }
int pos;
if (inputBuffer != null)
@@ -66,6 +70,7 @@ public class ReadScalars extends EvalFun
if (inputBag == null || inputBag.size() ==0)
{
log.warn("No scalar field to read, returning null");
+ valueLoaded = true;
return null;
} else if (inputBag.size() > 1) {
String msg = "Scalar has more than one row in the output.";
@@ -73,18 +78,19 @@ public class ReadScalars extends EvalFun
}
Tuple t1 = inputBag.iterator().next();
value = t1.get(pos);
+ valueLoaded = true;
return value;
}
-
+
ReadToEndLoader loader;
try {
pos = DataType.toInteger(input.get(0));
scalarfilename = DataType.toString(input.get(1));
-
+
// Hadoop security need this property to be set
Configuration conf = UDFContext.getUDFContext().getJobConf();
if (System.getenv("HADOOP_TOKEN_FILE_LOCATION") != null) {
- conf.set("mapreduce.job.credentials.binary",
+ conf.set("mapreduce.job.credentials.binary",
System.getenv("HADOOP_TOKEN_FILE_LOCATION"));
}
loader = new ReadToEndLoader(
@@ -97,19 +103,21 @@ public class ReadScalars extends EvalFun
Tuple t1 = loader.getNext();
if(t1 == null){
log.warn("No scalar field to read, returning null");
+ valueLoaded = true;
return null;
}
value = t1.get(pos);
Tuple t2 = loader.getNext();
if(t2 != null){
- String msg = "Scalar has more than one row in the output. "
+ String msg = "Scalar has more than one row in the output. "
+ "1st : " + t1 + ", 2nd :" + t2;
- throw new ExecException(msg);
+ throw new ExecException(msg);
}
-
+ valueLoaded = true;
+
} catch (Exception e) {
throw new ExecException(e.getMessage());
- }
+ }
}
return value;
}
@@ -117,5 +125,6 @@ public class ReadScalars extends EvalFun
public void setOutputBuffer(Map<String, DataBag> inputBuffer) {
this.inputBuffer = inputBuffer;
value = null;
+ valueLoaded = false;
}
}