You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by pr...@apache.org on 2014/03/06 01:29:22 UTC
svn commit: r1574729 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/newplan/logical/relational/LogicalPlanData.java
test/org/apache/pig/test/TestLogicalPlanBuilder.java
Author: prkommireddi
Date: Thu Mar 6 00:29:21 2014
New Revision: 1574729
URL: http://svn.apache.org/r1574729
Log:
PIG-3793: Provide info on number of LogicalRelationalOperator(s) used in the script through LogicalPlanData (prkommireddi)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/newplan/logical/relational/LogicalPlanData.java
pig/trunk/test/org/apache/pig/test/TestLogicalPlanBuilder.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1574729&r1=1574728&r2=1574729&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu Mar 6 00:29:21 2014
@@ -30,6 +30,8 @@ PIG-2207: Support custom counters for ag
IMPROVEMENTS
+PIG-3793: Provide info on number of LogicalRelationalOperator(s) used in the script through LogicalPlanData (prkommireddi)
+
PIG-3778: Log list of running jobs along with progress (rohini)
PIG-3675: Documentation for AccumuloStorage (elserj via daijy)
Modified: pig/trunk/src/org/apache/pig/newplan/logical/relational/LogicalPlanData.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/relational/LogicalPlanData.java?rev=1574729&r1=1574728&r2=1574729&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/relational/LogicalPlanData.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/relational/LogicalPlanData.java Thu Mar 6 00:29:21 2014
@@ -18,44 +18,95 @@
package org.apache.pig.newplan.logical.relational;
-import java.util.ArrayList;
+import java.util.Iterator;
import java.util.List;
+import org.apache.pig.classification.InterfaceAudience;
+import org.apache.pig.classification.InterfaceStability;
import org.apache.pig.newplan.Operator;
+import com.google.common.collect.Lists;
+
/**
*
- * This class provides information regarding the LogicalPlan. Make sure to
- * avoid exposing LogicalPlan itself. Only data regarding the logical plan
- * could be exposed but none of Pig internals (plans, operators etc) should
- * be.
- *
+ * This class provides information regarding the LogicalPlan. Make sure to avoid
+ * exposing LogicalPlan itself. Only data regarding the logical plan could be
+ * exposed but none of Pig internals (plans, operators etc) should be.
+ *
*/
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
public class LogicalPlanData {
// Never expose LogicalPlan
private final LogicalPlan lp;
+ private int numLogicalRelationalOperators;
+ // Sources and Sinks here refer to Load and Store operators
+ private final List<LOLoad> sources;
+ private final List<LOStore> sinks;
public LogicalPlanData(LogicalPlan lp) {
- if(lp == null) {
+ if (lp == null) {
throw new RuntimeException("LogicalPlan is null.");
}
- this.lp = lp;
+ this.lp = lp;
+ this.numLogicalRelationalOperators = 0;
+ this.sources = Lists.newArrayList();
+ this.sinks = Lists.newArrayList();
+ init();
+ }
+
+ private void init() {
+ Iterator<Operator> ops = this.lp.getOperators();
+
+ while (ops.hasNext()) {
+ Operator op = ops.next();
+ if (op instanceof LogicalRelationalOperator) {
+ this.numLogicalRelationalOperators++;
+ if (op instanceof LOLoad) {
+ sources.add((LOLoad) op);
+ } else if (op instanceof LOStore) {
+ sinks.add((LOStore) op);
+ }
+ }
+ }
}
/**
+ * Returns the number of {@link LogicalRelationalOperator}s present in the
+ * pig script.
+ *
+ * @return number of logical relational operators (Load, Join, Store etc)
*
- * @return This method return the list of source paths defined
- * in the script/query.
*/
- public List<String> getSources() {
- List<LOLoad> sources = getLOLoads();
- if (sources == null) {
- return null;
- }
+ public int getNumLogicalRelationOperators() {
+ return this.numLogicalRelationalOperators;
+ }
- List<String> result = new ArrayList<String>();
- for (LOLoad load : sources) {
+ /**
+ *
+ * @return number of Load statements in the script
+ */
+ public int getNumSources() {
+ return this.sources.size();
+ }
+
+ /**
+ *
+ * @return number of Store statements in the script
+ */
+ public int getNumSinks() {
+ return this.sinks.size();
+ }
+
+ /**
+ *
+ * @return This method return the list of Load paths defined in the
+ * script/query.
+ */
+ public List<String> getSources() {
+ List<String> result = Lists.newArrayList();
+ for (LOLoad load : this.sources) {
result.add(load.getFileSpec().getFileName());
}
@@ -67,12 +118,8 @@ public class LogicalPlanData {
* @return This method returns the list of store paths in the script/query.
*/
public List<String> getSinks() {
- List<LOStore> sinks = getLOStores();
- if (sinks == null) {
- return null;
- }
- List<String> result = new ArrayList<String>();
- for (LOStore sink : sinks) {
+ List<String> result = Lists.newArrayList();
+ for (LOStore sink : this.sinks) {
result.add(sink.getFileSpec().getFileName());
}
@@ -84,12 +131,8 @@ public class LogicalPlanData {
* @return This method returns the list of LoadFunc(s) used.
*/
public List<String> getLoadFuncs() {
- List<LOLoad> sources = getLOLoads();
- if (sources == null) {
- return null;
- }
- List<String> result = new ArrayList<String>();
- for (LOLoad load : sources) {
+ List<String> result = Lists.newArrayList();
+ for (LOLoad load : this.sources) {
result.add(load.getFileSpec().getFuncName());
}
@@ -101,56 +144,11 @@ public class LogicalPlanData {
* @return This method returns the list of StoreFunc(s) used.
*/
public List<String> getStoreFuncs() {
- List<LOStore> sinks = getLOStores();
- if (sinks == null) {
- return null;
- }
- List<String> storeFuncs = new ArrayList<String>();
- for (LOStore sink : sinks) {
+ List<String> storeFuncs = Lists.newArrayList();
+ for (LOStore sink : this.sinks) {
storeFuncs.add(sink.getFileSpec().getFuncName());
}
return storeFuncs;
}
-
- /**
- * Internal to Pig. Do not expose this method
- * @return
- */
- private List<LOLoad> getLOLoads() {
- List<Operator> sources = lp.getSources();
- if (sources == null) {
- return null;
- }
- List<LOLoad> result = new ArrayList<LOLoad>();
- for (Operator source : sources) {
- if (source instanceof LOLoad) {
- LOLoad load = (LOLoad) source;
- result.add(load);
- }
- }
-
- return result;
- }
-
- /**
- * Internal to Pig. Do not expose this method
- * @return
- */
- private List<LOStore> getLOStores() {
- List<Operator> sinks = lp.getSinks();
- if (sinks == null) {
- return null;
- }
- List<LOStore> result = new ArrayList<LOStore>();
- for (Operator sink : sinks) {
- if (sink instanceof LOStore) {
- LOStore store = (LOStore) sink;
- result.add(store);
- }
- }
-
- return result;
- }
-
}
Modified: pig/trunk/test/org/apache/pig/test/TestLogicalPlanBuilder.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestLogicalPlanBuilder.java?rev=1574729&r1=1574728&r2=1574729&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestLogicalPlanBuilder.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestLogicalPlanBuilder.java Thu Mar 6 00:29:21 2014
@@ -2168,9 +2168,11 @@ public class TestLogicalPlanBuilder {
LogicalPlanData lData = pigServer.getLogicalPlanData();
assertEquals("LoadFunc must be PigStorage", "org.apache.pig.builtin.PigStorage", lData.getLoadFuncs().get(0));
assertEquals("StoreFunc must be PigStorageWithSchema", "org.apache.pig.test.PigStorageWithSchema", lData.getStoreFuncs().get(0));
- assertEquals("Number of sources must be 2", lData.getSources().size(), 2);
+ assertEquals("Number of sources must be 2", lData.getNumSources(), 2);
+ assertEquals("Number of sinks must be 1", lData.getNumSinks(), 1);
assertTrue("Source must end with input.txt", lData.getSources().get(0).endsWith("input.txt"));
assertTrue("Sink must end with output", lData.getSinks().get(0).endsWith("output"));
+ assertEquals("Number of logical relational operators must be 4", lData.getNumLogicalRelationOperators(), 4);
}
/**