You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ch...@apache.org on 2013/03/05 18:32:50 UTC
svn commit: r1452913 - in /pig/trunk: ./ conf/ src/org/apache/pig/
src/org/apache/pig/parser/ test/org/apache/pig/test/
Author: cheolsoo
Date: Tue Mar 5 17:32:50 2013
New Revision: 1452913
URL: http://svn.apache.org/r1452913
Log:
PIG-3211: Allow default Load/Store funcs to be configurable (prkommireddi via cheolsoo)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/conf/pig.properties
pig/trunk/src/org/apache/pig/PigConfiguration.java
pig/trunk/src/org/apache/pig/parser/LogicalPlanBuilder.java
pig/trunk/src/org/apache/pig/parser/QueryParserUtils.java
pig/trunk/test/org/apache/pig/test/TestLogicalPlanBuilder.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1452913&r1=1452912&r2=1452913&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue Mar 5 17:32:50 2013
@@ -28,6 +28,8 @@ PIG-3174: Remove rpm and deb artifacts
IMPROVEMENTS
+PIG-3211: Allow default Load/Store funcs to be configurable (prkommireddi via cheolsoo)
+
PIG-3136: Introduce a syntax making declared aliases optional (jcoveney via cheolsoo)
PIG-3142: [piggybank] Fixed-width load and store functions for the Piggybank (jpacker via cheolsoo)
Modified: pig/trunk/conf/pig.properties
URL: http://svn.apache.org/viewvc/pig/trunk/conf/pig.properties?rev=1452913&r1=1452912&r2=1452913&view=diff
==============================================================================
--- pig/trunk/conf/pig.properties (original)
+++ pig/trunk/conf/pig.properties Tue Mar 5 17:32:50 2013
@@ -198,3 +198,18 @@ hcat.bin=/usr/local/hcat/bin/hcat
# to the same location. When set to true, stops the execution
# of script right away.
pig.location.check.strict=false
+
+######################################################################
+
+# This key is used to define the default load func. Pig will fallback
+# on PigStorage as default in case this is undefined.
+
+# pig.default.load.func=<fully qualified class name of a LoadFunc implementation>
+# For eg, pig.default.load.func=org.apache.pig.custom.MyCustomStorage
+
+# This key is used to define the default store func. Pig will fallback
+# on PigStorage as default in case this is undefined.
+
+# pig.default.store.func=<fully qualified class name of a StoreFunc implementation>
+# For eg, pig.default.store.func=org.apache.pig.custom.MyCustomStorage
+
Modified: pig/trunk/src/org/apache/pig/PigConfiguration.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/PigConfiguration.java?rev=1452913&r1=1452912&r2=1452913&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/PigConfiguration.java (original)
+++ pig/trunk/src/org/apache/pig/PigConfiguration.java Tue Mar 5 17:32:50 2013
@@ -81,4 +81,16 @@ public class PigConfiguration {
* will be set in the environment.
*/
public static final String PIG_STREAMING_ENVIRONMENT = "pig.streaming.environment";
+
+ /**
+ * This key is used to define the default load func. Pig will fallback on PigStorage
+ * as default in case this is undefined.
+ */
+ public static final String PIG_DEFAULT_LOAD_FUNC = "pig.default.load.func";
+
+ /**
+ * This key is used to define the default store func. Pig will fallback on PigStorage
+ * as default in case this is undefined.
+ */
+ public static final String PIG_DEFAULT_STORE_FUNC = "pig.default.store.func";
}
Modified: pig/trunk/src/org/apache/pig/parser/LogicalPlanBuilder.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/LogicalPlanBuilder.java?rev=1452913&r1=1452912&r2=1452913&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/LogicalPlanBuilder.java (original)
+++ pig/trunk/src/org/apache/pig/parser/LogicalPlanBuilder.java Tue Mar 5 17:32:50 2013
@@ -34,6 +34,7 @@ import org.antlr.runtime.RecognitionExce
import org.apache.pig.ExecType;
import org.apache.pig.FuncSpec;
import org.apache.pig.LoadFunc;
+import org.apache.pig.PigConfiguration;
import org.apache.pig.StoreFuncInterface;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
@@ -826,12 +827,10 @@ public class LogicalPlanBuilder {
String absolutePath;
LoadFunc loFunc;
try {
- FuncSpec instantiatedFuncSpec =
- funcSpec == null ?
- new FuncSpec(PigStorage.class.getName()) :
- funcSpec;
- loFunc = (LoadFunc)PigContext.instantiateFuncFromSpec(instantiatedFuncSpec);
- String fileNameKey = QueryParserUtils.constructFileNameSignature(filename, instantiatedFuncSpec) + "_" + (loadIndex++);
+ // Load LoadFunc class from default properties if funcSpec is null. Fallback on PigStorage if LoadFunc is not specified in properties.
+ funcSpec = funcSpec == null ? new FuncSpec(pigContext.getProperties().getProperty(PigConfiguration.PIG_DEFAULT_LOAD_FUNC, PigStorage.class.getName())) : funcSpec;
+ loFunc = (LoadFunc)PigContext.instantiateFuncFromSpec(funcSpec);
+ String fileNameKey = QueryParserUtils.constructFileNameSignature(filename, funcSpec) + "_" + (loadIndex++);
absolutePath = fileNameMap.get(fileNameKey);
if (absolutePath == null) {
absolutePath = loFunc.relativeToAbsolutePath( filename, QueryParserUtils.getCurrentDir( pigContext ) );
@@ -885,12 +884,9 @@ public class LogicalPlanBuilder {
String buildStoreOp(SourceLocation loc, String alias, String inputAlias, String filename, FuncSpec funcSpec)
throws ParserValidationException {
try {
- FuncSpec instantiatedFuncSpec =
- funcSpec == null ?
- new FuncSpec(PigStorage.class.getName()):
- funcSpec;
-
- StoreFuncInterface stoFunc = (StoreFuncInterface)PigContext.instantiateFuncFromSpec(instantiatedFuncSpec);
+ // Load StoreFunc class from default properties if funcSpec is null. Fallback on PigStorage if StoreFunc is not specified in properties.
+ funcSpec = funcSpec == null ? new FuncSpec(pigContext.getProperties().getProperty(PigConfiguration.PIG_DEFAULT_STORE_FUNC, PigStorage.class.getName())) : funcSpec;
+ StoreFuncInterface stoFunc = (StoreFuncInterface)PigContext.instantiateFuncFromSpec(funcSpec);
String fileNameKey = inputAlias + "_" + (storeIndex++) ;
String signature = inputAlias + "_" + newOperatorKey();
Modified: pig/trunk/src/org/apache/pig/parser/QueryParserUtils.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/QueryParserUtils.java?rev=1452913&r1=1452912&r2=1452913&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/QueryParserUtils.java (original)
+++ pig/trunk/src/org/apache/pig/parser/QueryParserUtils.java Tue Mar 5 17:32:50 2013
@@ -27,14 +27,13 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
-import org.antlr.runtime.tree.CommonTree;
-import org.antlr.runtime.tree.Tree;
import org.antlr.runtime.CommonTokenStream;
import org.antlr.runtime.RecognitionException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import org.antlr.runtime.tree.CommonTree;
+import org.antlr.runtime.tree.Tree;
import org.apache.hadoop.fs.Path;
import org.apache.pig.FuncSpec;
+import org.apache.pig.PigConfiguration;
import org.apache.pig.StoreFuncInterface;
import org.apache.pig.backend.datastorage.ContainerDescriptor;
import org.apache.pig.backend.datastorage.DataStorage;
@@ -49,7 +48,6 @@ import org.apache.pig.newplan.logical.re
import org.apache.pig.tools.pigstats.ScriptState;
public class QueryParserUtils {
- private static Log log = LogFactory.getLog( LogicalPlanGenerator.class );
public static String removeQuotes(String str) {
if (str.startsWith("\u005c'") && str.endsWith("\u005c'"))
@@ -60,9 +58,7 @@ public class QueryParserUtils {
public static void attachStorePlan(String scope, LogicalPlan lp, String fileName, String func,
Operator input, String alias, PigContext pigContext) throws FrontendException {
- if( func == null ) {
- func = PigStorage.class.getName();
- }
+ func = func == null ? pigContext.getProperties().getProperty(PigConfiguration.PIG_DEFAULT_STORE_FUNC, PigStorage.class.getName()) : func;
FuncSpec funcSpec = new FuncSpec( func );
StoreFuncInterface stoFunc = (StoreFuncInterface)PigContext.instantiateFuncFromSpec( funcSpec );
Modified: pig/trunk/test/org/apache/pig/test/TestLogicalPlanBuilder.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestLogicalPlanBuilder.java?rev=1452913&r1=1452912&r2=1452913&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestLogicalPlanBuilder.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestLogicalPlanBuilder.java Tue Mar 5 17:32:50 2013
@@ -33,6 +33,7 @@ import org.apache.pig.ExecType;
import org.apache.pig.FuncSpec;
import org.apache.pig.LoadCaster;
import org.apache.pig.LoadFunc;
+import org.apache.pig.PigConfiguration;
import org.apache.pig.PigException;
import org.apache.pig.PigServer;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit;
@@ -56,6 +57,7 @@ import org.apache.pig.newplan.logical.re
import org.apache.pig.newplan.logical.relational.LOGenerate;
import org.apache.pig.newplan.logical.relational.LOLoad;
import org.apache.pig.newplan.logical.relational.LOSort;
+import org.apache.pig.newplan.logical.relational.LOStore;
import org.apache.pig.newplan.logical.relational.LogicalPlan;
import org.apache.pig.newplan.logical.relational.LogicalSchema;
import org.apache.pig.test.utils.Identity;
@@ -2113,9 +2115,71 @@ public class TestLogicalPlanBuilder {
}
Assert.assertEquals("C", pigServer.getPigContext().getLastAlias());
}
-
- private void printPlan(LogicalExpressionPlan lp) {
- System.err.println( lp.toString() );
+
+ @Test
+ public void testBuildLoadOpWithDefaultFunc() throws Exception {
+ String query = "a = load '1.txt';" +
+ "store a into 'output';";
+ LogicalPlan lp = buildPlan(query);
+ FuncSpec funcSpec = getFirstLoadFuncSpec(lp);
+ assertEquals("org.apache.pig.builtin.PigStorage", funcSpec.getClassName());
+
+ // set default load func in config
+ pigServer.getPigContext().getProperties().setProperty(PigConfiguration.PIG_DEFAULT_LOAD_FUNC, "org.apache.pig.test.PigStorageWithSchema");
+ query = "a = load '1.txt';" +
+ "store a into 'output';";
+ lp = buildPlan(query);
+ funcSpec = getFirstLoadFuncSpec(lp);
+ assertEquals("org.apache.pig.test.PigStorageWithSchema", funcSpec.getClassName());
+
+ // unset default load func
+ pigServer.getPigContext().getProperties().remove(PigConfiguration.PIG_DEFAULT_LOAD_FUNC);
+ }
+
+ @Test
+ public void testBuildStoreOpWithDefaultFunc() throws Exception {
+ String query = "a = load '1.txt';" +
+ "store a into 'output';";
+ LogicalPlan lp = buildPlan(query);
+ FuncSpec funcSpec = getFirstStoreFuncSpec(lp);
+ assertEquals("org.apache.pig.builtin.PigStorage", funcSpec.getClassName());
+
+ // set default load func in config
+ pigServer.getPigContext().getProperties().setProperty(PigConfiguration.PIG_DEFAULT_STORE_FUNC, "org.apache.pig.test.PigStorageWithSchema");
+ query = "a = load '1.txt';" +
+ "store a into 'output';";
+ lp = buildPlan(query);
+ funcSpec = getFirstStoreFuncSpec(lp);
+ assertEquals("org.apache.pig.test.PigStorageWithSchema", funcSpec.getClassName());
+
+ // unset default load func
+ pigServer.getPigContext().getProperties().remove(PigConfiguration.PIG_DEFAULT_STORE_FUNC);
+ }
+
+ /**
+ * This method is not generic. Expects logical plan to have atleast
+ * 1 source and returns the corresponding FuncSpec.
+ * Specific to {@link #testBuildLoadOpWithDefaultFunc()}.
+ *
+ * @param lp LogicalPlan
+ * @return FuncSpec associated with 1st source
+ */
+ private FuncSpec getFirstLoadFuncSpec(LogicalPlan lp) {
+ List<Operator> sources = lp.getSources();
+ return ((LOLoad)sources.get(0)).getFileSpec().getFuncSpec();
+ }
+
+ /**
+ * This method is not generic. Expects logical plan to have atleast
+ * 1 sink and returns the corresponding FuncSpec
+ * Specific to {@link #testBuildStoreOpWithDefaultFunc()}.
+ *
+ * @param lp LogicalPlan
+ * @return FuncSpec associated with 1st sink
+ */
+ private FuncSpec getFirstStoreFuncSpec(LogicalPlan lp) {
+ List<Operator> sinks = lp.getSinks();
+ return ((LOStore)sinks.get(0)).getFileSpec().getFuncSpec();
}
private boolean checkPlanForProjectStar(LogicalExpressionPlan lp) {