You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by bi...@apache.org on 2012/10/12 02:47:56 UTC
svn commit: r1397399 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/tools/pigstats/PigStats.java
test/org/apache/pig/test/TestPigStats.java
Author: billgraham
Date: Fri Oct 12 00:47:55 2012
New Revision: 1397399
URL: http://svn.apache.org/viewvc?rev=1397399&view=rev
Log:
PIG-2964: Add helper method getJobList() to PigStats.JobGraph. Extend visibility of couple methods on same class (prkommireddi via billgraham)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java
pig/trunk/test/org/apache/pig/test/TestPigStats.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1397399&r1=1397398&r2=1397399&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri Oct 12 00:47:55 2012
@@ -25,6 +25,8 @@ PIG-1891 Enable StoreFunc to make intell
IMPROVEMENTS
+PIG-2964: Add helper method getJobList() to PigStats.JobGraph. Extend visibility of couple methods on same class (prkommireddi via billgraham)
+
PIG-2579: Support for multiple input schemas in AvroStorage (cheolsoo via sms)
PIG-2946: Documentation of "history" and "clear" commands (xalan via azaroth)
Modified: pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java?rev=1397399&r1=1397398&r2=1397399&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java (original)
+++ pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java Fri Oct 12 00:47:55 2012
@@ -25,6 +25,7 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
+import org.apache.commons.collections.IteratorUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.mapred.JobClient;
@@ -216,6 +217,17 @@ public abstract class PigStats {
return jp.toString();
}
+ /**
+ * Returns a List representation of the Job graph. Returned list is an
+ * ArrayList
+ *
+ * @return
+ */
+ @SuppressWarnings("unchecked")
+ public List<JobStats> getJobList() {
+ return IteratorUtils.toList(iterator());
+ }
+
public Iterator<JobStats> iterator() {
return new Iterator<JobStats>() {
private Iterator<Operator> iter = getOperators();
@@ -246,7 +258,7 @@ public abstract class PigStats {
return false;
}
- List<JobStats> getSuccessfulJobs() {
+ public List<JobStats> getSuccessfulJobs() {
ArrayList<JobStats> lst = new ArrayList<JobStats>();
Iterator<JobStats> iter = iterator();
while (iter.hasNext()) {
@@ -259,7 +271,7 @@ public abstract class PigStats {
return lst;
}
- List<JobStats> getFailedJobs() {
+ public List<JobStats> getFailedJobs() {
ArrayList<JobStats> lst = new ArrayList<JobStats>();
Iterator<JobStats> iter = iterator();
while (iter.hasNext()) {
Modified: pig/trunk/test/org/apache/pig/test/TestPigStats.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPigStats.java?rev=1397399&r1=1397398&r2=1397399&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPigStats.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPigStats.java Fri Oct 12 00:47:55 2012
@@ -29,6 +29,9 @@ import java.io.PrintWriter;
import junit.framework.Assert;
import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.pig.ExecType;
import org.apache.pig.PigServer;
@@ -41,11 +44,14 @@ import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.plan.OperatorKey;
import org.apache.pig.newplan.logical.relational.LogicalPlan;
import org.apache.pig.tools.pigstats.PigStats;
+import org.apache.pig.tools.pigstats.PigStats.JobGraph;
import org.apache.pig.tools.pigstats.ScriptState;
import org.junit.Test;
public class TestPigStats {
+ private static final Log LOG = LogFactory.getLog(TestPigStats.class);
+
@Test
public void testPigScriptInConf() throws Exception {
PrintWriter w = new PrintWriter(new FileWriter("test.pig"));
@@ -123,7 +129,8 @@ public class TestPigStats {
File outputFile = null;
try {
- outputFile = File.createTempFile("JIAR_1027", ".out");
+ String fileName = this.getClass().getName() + "_" + "testBytesWritten_JIRA_1027";
+ outputFile = File.createTempFile(fileName, ".out");
String filePath = outputFile.getAbsolutePath();
outputFile.delete();
PigServer pig = new PigServer(ExecType.LOCAL);
@@ -133,9 +140,8 @@ public class TestPigStats {
File dataFile = new File( outputFile.getAbsoluteFile() + File.separator + "part-00000" );
assertEquals(dataFile.length(), stats.getBytesWritten());
} catch (IOException e) {
- e.printStackTrace();
- System.err.println( e.getMessage() );
- fail("IOException happened");
+ LOG.error("Error while generating file", e);
+ fail("Encountered IOException");
} finally {
if (outputFile != null) {
// Hadoop Local mode creates a directory
@@ -181,18 +187,43 @@ public class TestPigStats {
}
}
- private void deleteDirectory( File dir ) {
- File[] files = dir.listFiles();
- for( File file : files ) {
- if( file.isDirectory() ) {
- deleteDirectory(file);
- } else {
- file.delete();
+ @Test
+ public void testPigStatsGetList() {
+ File outputFile = null;
+ try {
+ String filename = this.getClass().getSimpleName() + "_" + "testPigStatsGetList";
+ outputFile = File.createTempFile(filename, ".out");
+ String filePath = outputFile.getAbsolutePath();
+ outputFile.delete();
+ PigServer pigServer = new PigServer(ExecType.LOCAL);
+ pigServer.registerQuery("a = load 'test/org/apache/pig/test/data/passwd';");
+ pigServer.registerQuery("b = group a by $0;");
+ pigServer.registerQuery("c = foreach b generate group, COUNT(a) as cnt;");
+ pigServer.registerQuery("d = group c by cnt;");
+ pigServer.registerQuery("e = foreach d generate group;");
+ ExecJob job = pigServer.store("e", filePath);
+ JobGraph jobGraph = job.getStatistics().getJobGraph();
+ assertEquals(2, jobGraph.getJobList().size());
+
+ } catch (IOException e) {
+ LOG.error("IOException while creating file ", e);
+ fail("Encountered IOException");
+ } finally {
+ if (outputFile != null) {
+ // delete the directory before returning
+ deleteDirectory(outputFile);
}
}
- dir.delete();
}
+ private void deleteDirectory(File dir) {
+ try {
+ FileUtils.deleteDirectory(dir);
+ } catch (IOException e) {
+ LOG.error("Could not delete directory " + dir, e);
+ }
+ }
+
public static LogicalPlan getLogicalPlan(PigServer pig) throws Exception {
java.lang.reflect.Method buildLp = pig.getClass().getDeclaredMethod("buildLp");
buildLp.setAccessible(true);