You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ga...@apache.org on 2008/06/19 18:12:04 UTC
svn commit: r669525 [1/2] - in /incubator/pig/branches/types: ./ docs/
src/org/apache/pig/ src/org/apache/pig/tools/parameters/
test/org/apache/pig/test/ test/org/apache/pig/test/data/
Author: gates
Date: Thu Jun 19 09:12:03 2008
New Revision: 669525
URL: http://svn.apache.org/viewvc?rev=669525&view=rev
Log:
PIG-269 First batch of merges from trunk to branch. Thanks Pradeep.
Added:
incubator/pig/branches/types/docs/
incubator/pig/branches/types/docs/overview.html
incubator/pig/branches/types/src/org/apache/pig/tools/parameters/
incubator/pig/branches/types/src/org/apache/pig/tools/parameters/ParamLoader.jj
incubator/pig/branches/types/src/org/apache/pig/tools/parameters/ParameterSubstitutionPreprocessor.java
incubator/pig/branches/types/src/org/apache/pig/tools/parameters/PigFileParser.jj
incubator/pig/branches/types/src/org/apache/pig/tools/parameters/PreprocessorContext.java
incubator/pig/branches/types/test/org/apache/pig/test/TestParamSubPreproc.java
incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFile1.txt
incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFile2.txt
incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFile3.txt
incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFileDuplicates.txt
incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFileSameParamMultipleTimes.txt
incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFileWithEmptyComments.txt
incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFileWithInvalidLines.txt
incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFileWithValidLines.txt
incubator/pig/branches/types/test/org/apache/pig/test/data/ExpectedResult.pig
incubator/pig/branches/types/test/org/apache/pig/test/data/ExpectedResult2.pig
incubator/pig/branches/types/test/org/apache/pig/test/data/generate_date.sh
incubator/pig/branches/types/test/org/apache/pig/test/data/input1.pig
incubator/pig/branches/types/test/org/apache/pig/test/data/input2.pig
incubator/pig/branches/types/test/org/apache/pig/test/data/input3.pig
incubator/pig/branches/types/test/org/apache/pig/test/data/input4.pig
incubator/pig/branches/types/test/org/apache/pig/test/data/inputCmdlineParamPriortoDeclare.pig
incubator/pig/branches/types/test/org/apache/pig/test/data/inputCmdnameAsParamDeclare.pig
incubator/pig/branches/types/test/org/apache/pig/test/data/inputDefault.pig
incubator/pig/branches/types/test/org/apache/pig/test/data/inputEscape.pig
incubator/pig/branches/types/test/org/apache/pig/test/data/inputMultipleDeclares.pig
incubator/pig/branches/types/test/org/apache/pig/test/data/inputMultipleParams.pig
incubator/pig/branches/types/test/org/apache/pig/test/data/inputNoVars.pig
incubator/pig/branches/types/test/org/apache/pig/test/data/inputSubWithinLiteral.pig
incubator/pig/branches/types/test/org/apache/pig/test/data/inputSubstitutionWithinShellCommand.pig
incubator/pig/branches/types/test/org/apache/pig/test/data/inputSubstitutionWithinValue.pig
incubator/pig/branches/types/test/org/apache/pig/test/data/inputThreeParams.pig
incubator/pig/branches/types/test/org/apache/pig/test/data/newinputS.txt
incubator/pig/branches/types/test/org/apache/pig/test/data/output1.pig
incubator/pig/branches/types/test/org/apache/pig/test/data/output3.pig
Modified:
incubator/pig/branches/types/build.xml
incubator/pig/branches/types/src/org/apache/pig/Main.java
Modified: incubator/pig/branches/types/build.xml
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/build.xml?rev=669525&r1=669524&r2=669525&view=diff
==============================================================================
--- incubator/pig/branches/types/build.xml (original)
+++ incubator/pig/branches/types/build.xml Thu Jun 19 09:12:03 2008
@@ -16,8 +16,7 @@
<property name="src.dir" value="${basedir}/src/" />
<property name="src.lib.dir" value="${basedir}/lib-src/" />
<property name="src.gen.dir" value="${basedir}/src-gen/" />
-
- <property name="javacc.home" value="${basedir}/lib" />
+ <property name="src.docs.dir" value="${basedir}/docs/" />
<!-- javac properties -->
<property name="javac.debug" value="on" />
@@ -25,9 +24,9 @@
<property name="javac.deprecation" value="off" />
<property name="javac.version" value="1.5" />
<property name="javac.args" value="" />
- <!-- TODO we should use warning... <property name="javac.args.warnings" value="-Xlint:unchecked" /> -->
+ <!-- TODO we should use warning... <property name="javac.args.warnings" value="-Xlint:unchecked" /> -->
<property name="javac.args.warnings" value="" />
-
+
<!-- build properties -->
<property name="build.dir" value="${basedir}/build" />
<property name="build.classes" value="${build.dir}/classes" />
@@ -58,11 +57,17 @@
<property name="junit.hadoop.conf" value="" />
<property name="test.log.dir" value="${basedir}/test/logs"/>
<property name="junit.hadoop.conf" value="${user.home}/pigtest/conf/"/>
-
+ <property name="test.output" value="no"/>
+
<!-- javacc properties -->
<property name="src.gen.query.parser.dir" value="${src.gen.dir}/org/apache/pig/impl/logicalLayer/parser" />
<property name="src.gen.script.parser.dir" value="${src.gen.dir}/org/apache/pig/tools/pigscript/parser" />
+ <property name="src.gen.param.parser.dir" value="${src.gen.dir}/org/apache/pig/tools/parameters" />
<property name="src.gen.dot.parser.dir" value="${test.src.dir}/org/apache/pig/test/utils/dotGraph/parser" />
+ <property name="javacc.home" value="${basedir}/lib" />
+
+ <!-- javadoc properties -->
+ <property name="javadoc.link.java" value="http://java.sun.com/j2se/1.5.0/docs/api/" />
<!-- ====================================================== -->
<!-- Stuff needed by all targets -->
@@ -75,27 +80,52 @@
<fileset file="${lib.dir}/junit-4.1.jar" />
<fileset file="${lib.dir}/commons-collections-3.2.jar" />
</path>
-
+
<path id="test.classpath">
- <pathelement location="${build.classes}"/>
- <pathelement location="${test.src.dir}"/>
- <path refid="classpath"/>
+ <pathelement location="${build.classes}"/>
+ <pathelement location="${test.src.dir}"/>
+ <path refid="classpath"/>
</path>
-
+
<target name="init">
<mkdir dir="${src.gen.query.parser.dir}" />
<mkdir dir="${src.gen.script.parser.dir}" />
+ <mkdir dir="${src.gen.param.parser.dir}" />
<mkdir dir="${dist.dir}" />
<mkdir dir="${build.classes}" />
<mkdir dir="${test.build.classes}" />
<mkdir dir="${src.gen.dot.parser.dir}" />
- </target>
+ <tstamp>
+ <format property="timestamp" pattern="MMM dd yyyy, HH:mm:ss" />
+ </tstamp>
+ <svnversion outputproperty="svn.revision"/>
+ </target>
+
+ <macrodef name="svnversion">
+ <!-- the path needs to be small content otherwise it will take AGES ! -->
+ <attribute name="wcpath" default="${basedir}" />
+ <attribute name="outputproperty" />
+ <sequential>
+ <exec executable="svnversion" outputproperty="@{outputproperty}" failonerror="false" failifexecutionfails="false" >
+ <arg value="@{wcpath}" />
+ <redirector>
+ <outputfilterchain>
+ <tokenfilter>
+ <!-- version can be xxxx, xxxx:yyyy, xxxxM, xxxxS or xxxx:yyyyMS , ... just get the working copy one -->
+ <replaceregex pattern="((\d+).*)" replace="\2" />
+ </tokenfilter>
+ </outputfilterchain>
+ </redirector>
+ </exec>
+ </sequential>
+ </macrodef>
+
<!-- ================================================================== -->
<!-- Clean. Delete the build files, and their directories -->
<!-- ================================================================== -->
- <target name="clean">
+ <target name="clean" description="Cleanup build artifacts">
<delete dir="${src.gen.dir}" />
<delete dir="${build.dir}" />
<delete dir="${src.gen.dot.parser.dir}" />
@@ -104,10 +134,12 @@
<!-- ================================================================== -->
<!-- Java Compiler Compiler, generate Parsers -->
<!-- ================================================================== -->
- <target name="cc-compile" depends="init">
+ <target name="cc-compile" depends="init" description="Create and Compile Parser">
<jjtree target="${src.dir}/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt" outputdirectory="${src.gen.query.parser.dir}" javacchome="${javacc.home}" />
<javacc target="${src.gen.query.parser.dir}/QueryParser.jj" outputdirectory="${src.gen.query.parser.dir}" javacchome="${javacc.home}" />
<javacc target="${src.dir}/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj" outputdirectory="${src.gen.script.parser.dir}" javacchome="${javacc.home}" />
+ <javacc target="${src.dir}/org/apache/pig/tools/parameters/PigFileParser.jj" outputdirectory="${src.gen.param.parser.dir}" javacchome="${javacc.home}" />
+ <javacc target="${src.dir}/org/apache/pig/tools/parameters/ParamLoader.jj" outputdirectory="${src.gen.param.parser.dir}" javacchome="${javacc.home}" />
<jjtree target="${test.src.dir}/org/apache/pig/test/utils/dotGraph/Dot.jjt" outputdirectory="${src.gen.dot.parser.dir}" javacchome="${javacc.home}" />
<javacc target="${src.gen.dot.parser.dir}/Dot.jj" outputdirectory="${src.gen.dot.parser.dir}" javacchome="${javacc.home}" />
</target>
@@ -115,7 +147,7 @@
<!-- ================================================================== -->
<!-- Build sources -->
<!-- ================================================================== -->
- <target name="compile" depends="init, cc-compile">
+ <target name="compile" depends="init, cc-compile" description="Compile all artifacts">
<echo>*** Building Main Sources ***</echo>
<antcall target="compile-sources">
<param name="sources" value="${src.dir};${src.gen.dir};${src.lib.dir}/shock;${src.lib.dir}/bzip2" />
@@ -125,9 +157,6 @@
</target>
<target name="compile-test" depends="compile">
-
-
-
<echo>*** Building Test Sources ***</echo>
<antcall target="compile-sources">
<param name="sources" value="${test.src.dir}" />
@@ -149,14 +178,15 @@
<!-- ================================================================== -->
<!-- Documentation -->
<!-- ================================================================== -->
- <target name="javadoc" depends="init">
+ <target name="javadoc" depends="jar" description="Create documentation">
<mkdir dir="${build.javadoc}" />
- <javadoc overview="${src.dir}/overview.html" packagenames="org.apache.pig.*" destdir="${build.javadoc}" author="true" version="true" use="true" windowtitle="${Name} ${version} API" doctitle="${Name} ${version} API" bottom="Copyright &copy; ${year} The Apache Software Foundation">
+ <javadoc overview="${src.docs.dir}/overview.html" packagenames="org.apache.pig.*" destdir="${build.javadoc}" author="true" version="true" use="true" windowtitle="${Name} ${version} API" doctitle="${Name} ${version} API" bottom="Copyright &copy; ${year} The Apache Software Foundation">
<packageset dir="${src.dir}" />
<link href="${javadoc.link.java}" />
<classpath>
<path refid="classpath" />
<pathelement path="${java.class.path}" />
+ <pathelement path="${output.jarfile}" />
</classpath>
<group title="pig" packages="org.apache.*" />
</javadoc>
@@ -173,25 +203,47 @@
<!-- ================================================================== -->
<!-- TODO we should also exculte test here... -->
<!-- ================================================================== -->
- <target name="jar" depends="compile">
+ <target name="jar" depends="compile" description="Create pig jar">
+ <antcall target="jarWithSvn"/>
+ <antcall target="jarWithOutSvn"/>
+ </target>
+
+ <target name="jarWithSvn" if="svn.revision">
+ <antcall target="buildJar">
+ <param name="svnString" value="${svn.revision}" />
+ </antcall>
+ </target>
+
+ <target name="jarWithOutSvn" unless="svn.revision">
+ <antcall target="buildJar">
+ <param name="svnString" value=": unknown" />
+ </antcall>
+ </target>
+
+ <target name="buildJar">
+ <echo>svnString ${svnString}</echo>
<jar jarfile="${output.jarfile.core}" basedir="${build.classes}">
<manifest>
<attribute name="Main-Class" value="org.apache.pig.Main" />
<section name="org/apache/pig">
+ <attribute name="Implementation-Vendor" value="Apache" />
<attribute name="Implementation-Title" value="Pig" />
<attribute name="Implementation-Version" value="${version}" />
- <attribute name="Implementation-Vendor" value="Apache" />
+ <attribute name="Build-TimeStamp" value="${timestamp}" />
+ <attribute name="Svn-Revision" value="${svnString}" />
</section>
</manifest>
</jar>
- <!-- @depricated -->
+ <!-- @depricated -->
<jar jarfile="${output.jarfile}" basedir="${build.classes}">
<manifest>
<attribute name="Main-Class" value="org.apache.pig.Main" />
<section name="org/apache/pig">
+ <attribute name="Implementation-Vendor" value="Apache" />
<attribute name="Implementation-Title" value="Pig" />
<attribute name="Implementation-Version" value="${version}" />
- <attribute name="Implementation-Vendor" value="Apache" />
+ <attribute name="Build-TimeStamp" value="${timestamp}" />
+ <attribute name="Svn-Revision" value="${svnString}" />
</section>
</manifest>
<zipfileset src="${lib.dir}/junit-4.1.jar" />
@@ -221,9 +273,6 @@
</classpath>
<formatter type="${test.junit.output.format}" />
- <batchtest fork="yes" todir="${test.log.dir}" if="testcase">
- <fileset dir="test" includes="**/${testcase}.java"/>
- </batchtest>
<batchtest fork="yes" todir="${test.log.dir}" unless="testcase">
<fileset dir="test">
@@ -247,6 +296,9 @@
<exclude name="**/nightly/**" />
</fileset>
</batchtest>
+ <batchtest fork="yes" todir="${test.log.dir}" if="testcase">
+ <fileset dir="test" includes="**/${testcase}.java"/>
+ </batchtest>
</junit>
<fail if="tests.failed">Tests failed!</fail>
</target>
@@ -254,7 +306,7 @@
<!-- ================================================================== -->
<!-- D I S T R I B U T I O N -->
<!-- ================================================================== -->
- <target name="package" depends="jar, javadoc">
+ <target name="package" depends="jar, javadoc" description="Create a Pig release">
<mkdir dir="${dist.dir}" />
<mkdir dir="${dist.dir}/lib" />
<mkdir dir="${dist.dir}/scripts" />
@@ -298,7 +350,7 @@
<!-- ================================================================== -->
<!-- Make release tarball -->
<!-- ================================================================== -->
- <target name="tar" depends="package">
+ <target name="tar" depends="package" description="Create release tarball">
<tar compression="gzip" longfile="gnu" destfile="${build.dir}/${final.name}.tar.gz">
<tarfileset dir="${dist.dir}" mode="664">
<exclude name="scripts/*" />
Added: incubator/pig/branches/types/docs/overview.html
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/docs/overview.html?rev=669525&view=auto
==============================================================================
--- incubator/pig/branches/types/docs/overview.html (added)
+++ incubator/pig/branches/types/docs/overview.html Thu Jun 19 09:12:03 2008
@@ -0,0 +1,13 @@
+<HTML>
+<BODY>
+Provides the classes necessary to create programs in the high-level Pig Latin
+language and the infrastructure for evaluating these programs.
+<p>
+Pig is a platform for analyzing large data sets and consists of 2 layers:
+Pig's infrastructure layer consists of a compiler that produces sequences of
+Map-Reduce programs, for which large-scale parallel implementations already
+exist (e.g., the Hadoop project). Pig's language layer currently consists of
+a textual language called Pig Latin for expressing data analysis programs.
+</BODY>
+</HTML>
+
Modified: incubator/pig/branches/types/src/org/apache/pig/Main.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/Main.java?rev=669525&r1=669524&r2=669525&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/Main.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/Main.java Thu Jun 19 09:12:03 2008
@@ -35,10 +35,11 @@
import org.apache.pig.ExecType;
import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.logicalLayer.LogicalPlanBuilder;
+import org.apache.pig.impl.util.JarManager;
import org.apache.pig.tools.cmdline.CmdLineParser;
import org.apache.pig.tools.grunt.Grunt;
import org.apache.pig.tools.timer.PerformanceTimerFactory;
-
+import org.apache.pig.tools.parameters.ParameterSubstitutionPreprocessor;
public class Main
{
@@ -64,6 +65,7 @@
try {
BufferedReader in = null;
+ BufferedReader pin = null;
ExecMode mode = ExecMode.UNKNOWN;
int port = 0;
String file = null;
@@ -71,6 +73,10 @@
boolean brief = false;
String log4jconf = null;
boolean verbose = false;
+ boolean debug = false;
+ boolean dryrun = false;
+ ArrayList<String> params = new ArrayList<String>();
+ ArrayList<String> paramFiles = new ArrayList<String>();
CmdLineParser opts = new CmdLineParser(args);
// Don't use -l, --latest, -c, --cluster, -cp, -classpath, -D as these
@@ -86,6 +92,10 @@
opts.registerOpt('j', "jar", CmdLineParser.ValueExpected.REQUIRED);
opts.registerOpt('v', "verbose", CmdLineParser.ValueExpected.NOT_ACCEPTED);
opts.registerOpt('x', "exectype", CmdLineParser.ValueExpected.REQUIRED);
+ opts.registerOpt('i', "version", CmdLineParser.ValueExpected.OPTIONAL);
+ opts.registerOpt('p', "param", CmdLineParser.ValueExpected.OPTIONAL);
+ opts.registerOpt('m', "param_file", CmdLineParser.ValueExpected.OPTIONAL);
+ opts.registerOpt('r', "dryrun", CmdLineParser.ValueExpected.NOT_ACCEPTED);
char opt;
while ((opt = opts.getNextOpt()) != CmdLineParser.EndOfOpts) {
@@ -111,6 +121,7 @@
}
case 'd':
+ debug = true;
logLevel = Level.toLevel(opts.getValStr(), Level.INFO);
break;
@@ -137,6 +148,10 @@
break;
}
+ case 'm':
+ paramFiles.add(opts.getValStr());
+ break;
+
case 'o': {
String gateway = System.getProperty("ssh.gateway");
if (gateway == null || gateway.length() == 0) {
@@ -147,6 +162,17 @@
break;
}
+ case 'p':
+ String val = opts.getValStr();
+ params.add(opts.getValStr());
+ break;
+
+ case 'r':
+ // currently only used for parameter substitition
+ // will be extended in the future
+ dryrun = true;
+ break;
+
case 'v':
verbose = true;
break;
@@ -160,7 +186,9 @@
}
pigContext.setExecType(exectype);
break;
-
+ case 'i':
+ System.out.println(getVersionString());
+ return;
default: {
Character cc = new Character(opt);
throw new AssertionError("Unhandled option " + cc.toString());
@@ -203,15 +231,26 @@
// TODO Add a file appender for the logs
// TODO Need to create a property in the properties file for it.
- // Don't forget to undo all this for the port option.
-
- // I might know what I want to do next, then again I might not.
+ // construct the parameter subsitution preprocessor
Grunt grunt = null;
+ String substFile = null;
switch (mode) {
case FILE:
// Run, using the provided file as a pig file
in = new BufferedReader(new FileReader(file));
- grunt = new Grunt(in, pigContext);
+
+ // run parameter substition preoprocessor first
+ substFile = file + ".substituted";
+ pin = runParamPreprocessor(in, params, paramFiles, substFile, debug || dryrun);
+ if (dryrun){
+ log.info("Dry run completed. Substitued pig script is at " + substFile);
+ return;
+ }
+
+ if (!debug)
+ new File(substFile).deleteOnExit();
+
+ grunt = new Grunt(pin, pigContext);
grunt.exec();
return;
@@ -229,7 +268,7 @@
grunt.exec();
rc = 0;
return;
- }
+ }
default:
break;
@@ -256,7 +295,19 @@
}
mode = ExecMode.FILE;
in = new BufferedReader(new FileReader(remainders[0]));
- grunt = new Grunt(in, pigContext);
+
+ // run parameter substition preoprocessor first
+ substFile = remainders[0] + ".substituted";
+ pin = runParamPreprocessor(in, params, paramFiles, substFile, debug || dryrun);
+ if (dryrun){
+ log.info("Dry run completed. Substitued pig script is at " + substFile);
+ return;
+ }
+
+ if (!debug)
+ new File(substFile).deleteOnExit();
+
+ grunt = new Grunt(pin, pigContext);
grunt.exec();
rc = 0;
return;
@@ -276,9 +327,52 @@
System.exit(rc);
}
}
+
+// retruns the stream of final pig script to be passed to Grunt
+private static BufferedReader runParamPreprocessor(BufferedReader origPigScript, ArrayList<String> params,
+ ArrayList<String> paramFiles, String scriptFile, boolean createFile)
+ throws org.apache.pig.tools.parameters.ParseException, IOException{
+ ParameterSubstitutionPreprocessor psp = new ParameterSubstitutionPreprocessor(50);
+ String[] type1 = new String[1];
+ String[] type2 = new String[1];
+
+ if (createFile){
+ BufferedWriter fw = new BufferedWriter(new FileWriter(scriptFile));
+ psp.genSubstitutedFile (origPigScript, fw, params.size() > 0 ? params.toArray(type1) : null,
+ paramFiles.size() > 0 ? paramFiles.toArray(type2) : null);
+ return new BufferedReader(new FileReader (scriptFile));
+
+ } else {
+ StringWriter writer = new StringWriter();
+ psp.genSubstitutedFile (origPigScript, writer, params.size() > 0 ? params.toArray(type1) : null,
+ paramFiles.size() > 0 ? paramFiles.toArray(type2) : null);
+ return new BufferedReader(new StringReader(writer.toString()));
+ }
+}
+private static String getVersionString() {
+ String findContainingJar = JarManager.findContainingJar(Main.class);
+ try {
+ StringBuffer buffer = new StringBuffer();
+ JarFile jar = new JarFile(findContainingJar);
+ final Manifest manifest = jar.getManifest();
+ final Map <String,Attributes> attrs = manifest.getEntries();
+ Attributes attr = attrs.get("org/apache/pig");
+ String version = (String) attr.getValue("Implementation-Version");
+ String svnRevision = (String) attr.getValue("Svn-Revision");
+ String buildTime = (String) attr.getValue("Build-TimeStamp");
+ // we use a version string similar to svn
+ //svn, version 1.4.4 (r25188)
+ // compiled Sep 23 2007, 22:32:34
+ return "Apache Pig version " + version + " (r" + svnRevision + ") \ncompiled "+buildTime;
+ } catch (Exception e) {
+ throw new RuntimeException("unable to read pigs manifest file", e);
+ }
+}
+
public static void usage()
{
+ System.out.println("\n"+getVersionString()+"\n");
System.out.println("USAGE: Pig [options] [-] : Run interactively in grunt shell.");
System.out.println(" Pig [options] -e[xecute] cmd [cmd ...] : Run cmd(s).");
System.out.println(" Pig [options] [-f[ile]] file : Run cmds found in file.");
@@ -292,5 +386,6 @@
System.out.println(" -o, -hod read hod server from system property ssh.gateway");
System.out.println(" -v, -verbose print all log messages to screen (default to print only INFO and above to screen)");
System.out.println(" -x, -exectype local|mapreduce, mapreduce is default");
+ System.out.println(" -i, -version display version information");
}
}
Added: incubator/pig/branches/types/src/org/apache/pig/tools/parameters/ParamLoader.jj
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/tools/parameters/ParamLoader.jj?rev=669525&view=auto
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/tools/parameters/ParamLoader.jj (added)
+++ incubator/pig/branches/types/src/org/apache/pig/tools/parameters/ParamLoader.jj Thu Jun 19 09:12:03 2008
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This grammar is used to process parameters passed from
+ * command line or config file
+ */
+
+options {
+ // Generate non-static functions
+ STATIC = false;
+}
+PARSER_BEGIN(ParamLoader)
+package org.apache.pig.tools.parameters;
+
+import java.io.IOException;
+import java.util.Hashtable;
+
+public class ParamLoader {
+
+ private PreprocessorContext pc;
+ public void setContext(PreprocessorContext pc) {
+ this.pc = pc;
+ }
+
+ private static String unquote(String s)
+ {
+ if (s.charAt(0) == '\'' && s.charAt(s.length()-1) == '\'')
+ return s.substring(1, s.length()-1);
+ else if (s.charAt(0) == '"' && s.charAt(s.length()-1) == '"')
+ return s.substring(1, s.length()-1);
+ else
+ return s;
+ }
+}
+
+PARSER_END(ParamLoader)
+
+SKIP :
+{
+ "\n"
+| "\r"
+| " "
+| "\t"
+}
+
+TOKEN:
+{
+ <#LETTER : ["a"-"z", "A"-"Z"] >
+ |
+ <#DIGIT : ["0"-"9"] >
+ |
+ <#SPECIALCHAR : ["_"] >
+}
+
+TOKEN :
+{
+// <IDENTIFIER: (<LETTER> | <SPECIALCHAR>) (<DIGIT> | <LETTER> | <SPECIALCHAR>)*>
+ <IDENTIFIER: (<SPECIALCHAR>)*<LETTER>(<DIGIT> | <LETTER> | <SPECIALCHAR>)*>
+ |
+ <ORD_LINE: ~["\"" , "'" , "`" , "a"-"z" , "A"-"Z" , "_" , "#" , "=" , " " , "\n" , "\t" , "\r"] (~["\n","\r"])* >
+ |
+ <LITERAL: ("\"" ((~["\""])*("\\\"")?)* "\"")|("'" ((~["'"])*("\\\'")?)* "'") >
+ |
+ <SHELLCMD: "`" (~["`"])* "`" >
+ |
+ <EQUALS: "=" >
+ |
+ <COMMENT: "#" (~["\n"])* ("\n"|"\r") >
+
+}
+
+
+
+boolean Parse() throws IOException :
+{
+ Token id, val;
+ boolean more=true;
+ String s;
+}
+{
+ (
+ (
+ id=<IDENTIFIER>
+ <EQUALS>
+ (
+ val=<IDENTIFIER> {pc.processOrdLine(id.image , val.image);}
+ |
+ val=<ORD_LINE> {pc.processOrdLine(id.image , val.image);}
+ |
+ val=<SHELLCMD> { pc.processShellCmd(id.image , val.image); }
+ |
+ val=<LITERAL> { s = unquote(val.image); pc.processOrdLine(id.image , s); }
+ )
+ )
+ |
+ <COMMENT> { }
+ |
+ <EOF> { more=false; }
+ ) { return more; }
+
+
+}
+
+
Added: incubator/pig/branches/types/src/org/apache/pig/tools/parameters/ParameterSubstitutionPreprocessor.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/tools/parameters/ParameterSubstitutionPreprocessor.java?rev=669525&view=auto
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/tools/parameters/ParameterSubstitutionPreprocessor.java (added)
+++ incubator/pig/branches/types/src/org/apache/pig/tools/parameters/ParameterSubstitutionPreprocessor.java Thu Jun 19 09:12:03 2008
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This is the main driver for parameter substitution
+ * http://wiki.apache.org/pig/ParameterSubstitution
+ */
+package org.apache.pig.tools.parameters;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.log4j.Appender;
+import org.apache.log4j.Logger;
+
+import java.util.Hashtable;
+import java.io.FileInputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.BufferedReader;
+import java.io.StringReader;
+import java.io.Writer;
+
+public class ParameterSubstitutionPreprocessor {
+
+ private PreprocessorContext pc;
+ private final Log log = LogFactory.getLog(getClass());
+ private ParamLoader paramParser;
+ private PigFileParser pigParser;
+
+ /**
+ * @param limit - max number of parameters to expect. Smaller values would
+ * would not cause incorrect behavior but would impact performance
+ */
+ public ParameterSubstitutionPreprocessor(int limit) {
+ pc = new PreprocessorContext(50);
+ StringReader sr = null;
+
+ paramParser = new ParamLoader(sr);
+ paramParser.setContext(pc);
+
+ pigParser = new PigFileParser(sr);
+ pigParser.setContext(pc);
+ }
+
+ /**
+ * This is the main API that takes script template and produces pig script
+ * @param pigInput - input stream that contains pig file
+ * @param pigOutput - stream where transformed file is written
+ * @param args - command line arguments in the order they appear on the command line; format: key=val
+ * @param argFiles - list of configuration files in the order they appear on the command line
+ */
+ public void genSubstitutedFile (BufferedReader pigInput, Writer pigOutput, String[] args, String[] argFiles) throws ParseException {
+
+ // load parameters from the files followed by parameters
+ // from command line both in the order they appear on the command line
+ // this enforces precedence rules
+ if (argFiles!=null) {
+ for (int i=0;i<argFiles.length;i++) {
+ if (argFiles[i].length() > 0)
+ loadParamsFromFile(argFiles[i]);
+ }
+ }
+ if (args!=null) {
+ for (int i=0;i<args.length;i++) {
+ if (args[i].length() > 0)
+ loadParamsFromCmdline(args[i]);
+ }
+ }
+
+ // perform the substitution
+ parsePigFile(pigInput , pigOutput);
+ }
+
+ private void parsePigFile(BufferedReader in, Writer out) throws ParseException {
+
+ try {
+ String line;
+ int lineNum=0;
+
+ while((line = in.readLine())!= null)
+ {
+ lineNum++;
+
+ if (line.length() == 0) {
+ out.append("\n");
+ continue;
+ }
+
+ if ( !(line.startsWith("%")) ) {
+ //process an ordinary pig line - perform substitution
+ String sub_line = pc.substitute(line);
+ out.append(sub_line);
+ out.append("\n");
+ continue;
+ }
+
+ //parse the declare/default line
+ pigParser.ReInit(new StringReader(line));
+ pigParser.Parse();
+ }
+
+ //close input and output streams
+ in.close();
+ out.flush();
+ out.close();
+ } catch (IOException e) {
+ RuntimeException rte = new RuntimeException(e.getMessage() , e);
+ throw rte;
+ }
+
+ }
+
+ /*
+ * populates the param-val hashtable with parameters read from a file
+ * @param filename - name of the config file
+ */
+ private void loadParamsFromFile(String filename) throws ParseException {
+
+ try {
+ BufferedReader in = new BufferedReader(new FileReader(filename));
+ String line;
+
+ paramParser.ReInit(in);
+ while (paramParser.Parse()) {}
+ in.close();
+ } catch (IOException e) {
+ RuntimeException rte = new RuntimeException(e.getMessage() , e);
+ throw rte;
+ }
+
+ }
+
+ /*
+ * adds key-val pairs from cmd line to the param-val hashtable
+ * @param param contains key-val of the form key='value'
+ */
+ private void loadParamsFromCmdline(String line) throws ParseException {
+ try {
+ // new lines are needed by the parser
+ paramParser.ReInit(new StringReader(line));
+ paramParser.Parse();
+ } catch (IOException e) {
+ RuntimeException rte = new RuntimeException(e.getMessage() , e);
+ throw rte;
+ }
+
+ }
+
+
+}
Added: incubator/pig/branches/types/src/org/apache/pig/tools/parameters/PigFileParser.jj
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/tools/parameters/PigFileParser.jj?rev=669525&view=auto
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/tools/parameters/PigFileParser.jj (added)
+++ incubator/pig/branches/types/src/org/apache/pig/tools/parameters/PigFileParser.jj Thu Jun 19 09:12:03 2008
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This grammar is used to process %declare and %default commands
+ * within pig script
+ */
+
+options {
+ // Generate non-static functions
+ STATIC = false;
+ IGNORE_CASE = true;
+}
+PARSER_BEGIN(PigFileParser)
+package org.apache.pig.tools.parameters;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Hashtable;
+
+public class PigFileParser {
+ private PreprocessorContext pc;
+ public void setContext(PreprocessorContext pc) {
+ this.pc = pc;
+ }
+
+ private static String unquote(String s)
+ {
+ if (s.charAt(0) == '\'' && s.charAt(s.length()-1) == '\'')
+ return s.substring(1, s.length()-1);
+ else if (s.charAt(0) == '"' && s.charAt(s.length()-1) == '"')
+ return s.substring(1, s.length()-1);
+ else
+ return s;
+ }
+}
+
+PARSER_END(PigFileParser)
+
+SKIP :
+{
+ "\n"
+| "\r"
+| " "
+| "\t"
+}
+
+TOKEN:
+{
+ <#LETTER : ["a"-"z", "A"-"Z"] >
+ |
+ <#DIGIT : ["0"-"9"] >
+ |
+ <#SPECIALCHAR : ["_"] >
+}
+
+TOKEN :
+{
+ <DECLARE: "%declare" >
+ |
+ <PIGDEFAULT: "%default" >
+}
+
+TOKEN :
+{
+ <IDENTIFIER: (<SPECIALCHAR>)*<LETTER>(<DIGIT> | <LETTER> | <SPECIALCHAR>)*>
+ |
+ <LITERAL: ("\"" ((~["\""])*("\\\"")?)* "\"")|("'" ((~["'"])*("\\\'")?)* "'") >
+ |
+ <SHELLCMD: "`" (~["`"])* "`" >
+ |
+ <ORD_LINE: ~["\"" , "'" , "`" , "a"-"z" , "A"-"Z" , "_" , "#" , "=" , " " , "\n" , "\t" , "\r", "%"] (~["\n","\r"])* >
+}
+
+
+
+void Parse() throws IOException :
+{
+ Token id, val;
+ Boolean overwrite;
+ String s;
+}
+{
+
+ <DECLARE> { overwrite=true; }
+ (
+ id=<IDENTIFIER>
+ (
+ val=<IDENTIFIER> {pc.processOrdLine(id.image , val.image, overwrite);}
+ |
+ val=<ORD_LINE> {pc.processOrdLine(id.image , val.image, overwrite);}
+ |
+ val=<SHELLCMD> { pc.processShellCmd(id.image , val.image, overwrite); }
+ |
+ val=<LITERAL> { s = unquote(val.image); pc.processOrdLine(id.image, s, overwrite); }
+
+ )
+ )
+ |
+ <PIGDEFAULT> { overwrite=false; }
+ (
+ id=<IDENTIFIER>
+ (
+ val=<IDENTIFIER> {pc.processOrdLine(id.image , val.image, overwrite);}
+ |
+ val=<ORD_LINE> {pc.processOrdLine(id.image , val.image, overwrite);}
+ |
+ val=<SHELLCMD> { pc.processShellCmd(id.image , val.image , overwrite); }
+ |
+ val=<LITERAL> { s = unquote(val.image); pc.processOrdLine(id.image , s, overwrite); }
+ )
+ )
+}
Added: incubator/pig/branches/types/src/org/apache/pig/tools/parameters/PreprocessorContext.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/tools/parameters/PreprocessorContext.java?rev=669525&view=auto
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/tools/parameters/PreprocessorContext.java (added)
+++ incubator/pig/branches/types/src/org/apache/pig/tools/parameters/PreprocessorContext.java Thu Jun 19 09:12:03 2008
@@ -0,0 +1,236 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This is helper class for parameter substitution
+ */
+
+package org.apache.pig.tools.parameters;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.log4j.Logger;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.BufferedReader;
+import java.util.Hashtable;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class PreprocessorContext {
+
+ private Hashtable<String , String> param_val ;
+
+ private final Log log = LogFactory.getLog(getClass());
+
+ /**
+ * @param limit - max number of parameters. Passing
+ * smaller number only impacts performance
+ */
+ public PreprocessorContext(int limit){
+ param_val = new Hashtable<String, String> (limit);
+ }
+
+ /*
+ public void processLiteral(String key, String val) {
+ processLiteral(key, val, true);
+ } */
+
+ /**
+ * This method generates parameter value by running specified command
+ *
+ * @param key - parameter name
+ * @param val - string containing command to be executed
+ */
+ public void processShellCmd(String key, String val) {
+ processShellCmd(key, val, true);
+ }
+
+ /**
+ * This method generates value for the specified key by
+ * performing substitution if needed within the value first.
+ *
+ * @param key - parameter name
+ * @param val - value supplied for the key
+ */
+ public void processOrdLine(String key, String val) {
+ processOrdLine(key, val, true);
+ }
+
+ /*
+ public void processLiteral(String key, String val, Boolean overwrite) {
+
+ if (param_val.containsKey(key)) {
+ if (overwrite) {
+ log.warn("Warning : Multiple values found for " + key + ". Using value " + val);
+ } else {
+ return;
+ }
+ }
+
+ String sub_val = substitute(val);
+ param_val.put(key, sub_val);
+ } */
+
+ /**
+ * This method generates parameter value by running specified command
+ *
+ * @param key - parameter name
+ * @param val - string containing command to be executed
+ */
+ public void processShellCmd(String key, String val, Boolean overwrite) {
+
+ if (param_val.containsKey(key)) {
+ if (overwrite) {
+ log.warn("Warning : Multiple values found for " + key + ". Using value " + val);
+ } else {
+ return;
+ }
+ }
+
+ val=val.substring(1, val.length()-1); //to remove the backticks
+ String sub_val = substitute(val);
+ sub_val = executeShellCommand(sub_val);
+ param_val.put(key, sub_val);
+ }
+
+ /**
+ * This method generates value for the specified key by
+ * performing substitution if needed within the value first.
+ *
+ * @param key - parameter name
+ * @param val - value supplied for the key
+ * @param overwide - specifies whether the value should be replaced if it already exists
+ */
+ public void processOrdLine(String key, String val, Boolean overwrite) {
+
+ if (param_val.containsKey(key)) {
+ if (overwrite) {
+ log.warn("Warning : Multiple values found for " + key + ". Using value " + val);
+ } else {
+ return;
+ }
+ }
+
+ String sub_val = substitute(val);
+ param_val.put(key, sub_val);
+ }
+
+
+ /*
+ * executes the 'cmd' in shell and returns result
+ */
+ private String executeShellCommand (String cmd)
+ {
+ Process p;
+ String streamData="";
+ String streamError="";
+ try {
+ log.info("Executing command : " + cmd);
+ // we can't use exec directly since it does not handle
+ // case like foo -c "bar bar" correctly. It splits on white spaces even in presents of quotes
+ String[] cmdArgs = new String[3];
+ cmdArgs[0] = "bash";
+ cmdArgs[1] = "-c";
+ StringBuffer sb = new StringBuffer("exec ");
+ sb.append(cmd);
+ cmdArgs[2] = sb.toString();
+
+ p = Runtime.getRuntime().exec(cmdArgs);
+
+ } catch (IOException e) {
+ RuntimeException rte = new RuntimeException("IO Exception while executing shell command : "+e.getMessage() , e);
+ throw rte;
+ }
+
+ int exitVal;
+ try {
+ exitVal = p.waitFor();
+ } catch (InterruptedException e) {
+ RuntimeException rte = new RuntimeException("Interrupted Thread Exception while waiting for command to get over"+e.getMessage() , e);
+ throw rte;
+ }
+
+ if (exitVal != 0) {
+ RuntimeException rte = new RuntimeException("Error executing shell command: " + cmd + ". Command exit with exit code of " + exitVal );
+ throw rte;
+ }
+
+ try{
+ InputStreamReader isr = new InputStreamReader(p.getInputStream());
+ BufferedReader br = new BufferedReader(isr);
+ String line=null;
+ while ( (line = br.readLine()) != null){
+ streamData+=(line+"\n");
+ }
+ } catch (IOException e){
+ RuntimeException rte = new RuntimeException("IO Exception while executing shell command : "+e.getMessage() , e);
+ throw rte;
+ }
+
+ try {
+ InputStreamReader isr = new InputStreamReader(p.getErrorStream());
+ BufferedReader br = new BufferedReader(isr);
+ String line=null;
+ while ( (line = br.readLine()) != null ) {
+ streamError += (line+"\n");
+ }
+ log.debug("Error stream while executing shell command : " + streamError);
+ } catch (Exception e) {
+ RuntimeException rte = new RuntimeException("IO Exception while executing shell command : "+e.getMessage() , e);
+ throw rte;
+ }
+
+ return streamData.trim();
+ }
+
+ private static String id_regex = "\\$[_]*[a-zA-Z][a-zA-Z_0-9]*";
+ public String substitute(String line) {
+
+ int index = line.indexOf('$');
+ if (index == -1) return line;
+
+ String replaced_line = new String(line);
+
+ Pattern identifier = Pattern.compile( id_regex );
+ Matcher keyMatcher = identifier.matcher( line );
+ String key="";
+ String val="";
+
+ while (keyMatcher.find()) {
+ // make sure that we don't perform parameter substitution
+ // for escaped vars of the form \$<id>
+ if ( (keyMatcher.start() == 0) || (line.charAt( keyMatcher.start() - 1)) != '\\' ) {
+ key = keyMatcher.group().substring(1); //skip the '$'
+ if (!(param_val.containsKey(key))) {
+ throw new RuntimeException("Undefined parameter : "+key);
+ }
+ val = param_val.get(key);
+ //String litVal = Matcher.quoteReplacement(val);
+ replaced_line = replaced_line.replaceFirst("\\$"+key, val);
+ }
+ }
+
+ // unescape $<id>
+ replaced_line = replaced_line.replaceAll("\\\\\\$","\\$");
+ return replaced_line;
+ }
+
+}
+
+