You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by na...@apache.org on 2009/08/05 01:10:55 UTC
svn commit: r801012 - in /hadoop/hive/trunk: ./ ant/ bin/ext/
cli/src/java/org/apache/hadoop/hive/cli/
contrib/src/java/org/apache/hadoop/hive/contrib/fileformat/base64/
data/conf/ eclipse-templates/ hwi/src/java/org/apache/hadoop/hive/hwi/ ql/
ql/src/...
Author: namit
Date: Tue Aug 4 23:10:53 2009
New Revision: 801012
URL: http://svn.apache.org/viewvc?rev=801012&view=rev
Log:
HIVE-487. Make hive compatibale with hadoop 20
(Todd Lipcon via namit)
Added:
hadoop/hive/trunk/shims/
hadoop/hive/trunk/shims/build.xml
hadoop/hive/trunk/shims/ivy.xml
hadoop/hive/trunk/shims/src/
hadoop/hive/trunk/shims/src/0.17/
hadoop/hive/trunk/shims/src/0.17/java/
hadoop/hive/trunk/shims/src/0.17/java/org/
hadoop/hive/trunk/shims/src/0.17/java/org/apache/
hadoop/hive/trunk/shims/src/0.17/java/org/apache/hadoop/
hadoop/hive/trunk/shims/src/0.17/java/org/apache/hadoop/hive/
hadoop/hive/trunk/shims/src/0.17/java/org/apache/hadoop/hive/shims/
hadoop/hive/trunk/shims/src/0.17/java/org/apache/hadoop/hive/shims/Hadoop17Shims.java
hadoop/hive/trunk/shims/src/0.17/java/org/apache/hadoop/hive/shims/Jetty17Shims.java
hadoop/hive/trunk/shims/src/0.18/
hadoop/hive/trunk/shims/src/0.18/java/
hadoop/hive/trunk/shims/src/0.18/java/org/
hadoop/hive/trunk/shims/src/0.18/java/org/apache/
hadoop/hive/trunk/shims/src/0.18/java/org/apache/hadoop/
hadoop/hive/trunk/shims/src/0.18/java/org/apache/hadoop/hive/
hadoop/hive/trunk/shims/src/0.18/java/org/apache/hadoop/hive/shims/
hadoop/hive/trunk/shims/src/0.18/java/org/apache/hadoop/hive/shims/Hadoop18Shims.java
hadoop/hive/trunk/shims/src/0.18/java/org/apache/hadoop/hive/shims/Jetty18Shims.java
hadoop/hive/trunk/shims/src/0.19/
hadoop/hive/trunk/shims/src/0.19/java/
hadoop/hive/trunk/shims/src/0.19/java/org/
hadoop/hive/trunk/shims/src/0.19/java/org/apache/
hadoop/hive/trunk/shims/src/0.19/java/org/apache/hadoop/
hadoop/hive/trunk/shims/src/0.19/java/org/apache/hadoop/hive/
hadoop/hive/trunk/shims/src/0.19/java/org/apache/hadoop/hive/shims/
hadoop/hive/trunk/shims/src/0.19/java/org/apache/hadoop/hive/shims/Hadoop19Shims.java
hadoop/hive/trunk/shims/src/0.19/java/org/apache/hadoop/hive/shims/Jetty19Shims.java
hadoop/hive/trunk/shims/src/0.20/
hadoop/hive/trunk/shims/src/0.20/java/
hadoop/hive/trunk/shims/src/0.20/java/org/
hadoop/hive/trunk/shims/src/0.20/java/org/apache/
hadoop/hive/trunk/shims/src/0.20/java/org/apache/hadoop/
hadoop/hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/
hadoop/hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/
hadoop/hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
hadoop/hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Jetty20Shims.java
hadoop/hive/trunk/shims/src/common/
hadoop/hive/trunk/shims/src/common/java/
hadoop/hive/trunk/shims/src/common/java/org/
hadoop/hive/trunk/shims/src/common/java/org/apache/
hadoop/hive/trunk/shims/src/common/java/org/apache/hadoop/
hadoop/hive/trunk/shims/src/common/java/org/apache/hadoop/hive/
hadoop/hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/
hadoop/hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java
hadoop/hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/JettyShims.java
hadoop/hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/ShimLoader.java
Modified:
hadoop/hive/trunk/CHANGES.txt
hadoop/hive/trunk/ant/build.xml
hadoop/hive/trunk/bin/ext/cli.sh
hadoop/hive/trunk/build-common.xml
hadoop/hive/trunk/build.xml
hadoop/hive/trunk/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java
hadoop/hive/trunk/contrib/src/java/org/apache/hadoop/hive/contrib/fileformat/base64/Base64TextInputFormat.java
hadoop/hive/trunk/data/conf/hive-log4j.properties
hadoop/hive/trunk/eclipse-templates/.classpath
hadoop/hive/trunk/hwi/src/java/org/apache/hadoop/hive/hwi/HWIServer.java
hadoop/hive/trunk/ql/build.xml
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapRedTask.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java
Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=801012&r1=801011&r2=801012&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Tue Aug 4 23:10:53 2009
@@ -109,6 +109,9 @@
HIVE-607. Add udafs for variance and standard deviation
(Emil Ibrishimov via namit)
+ HIVE-487. Make hive compatibale with hadoop 20
+ (Todd Lipcon via namit)
+
IMPROVEMENTS
HIVE-389. Option to build without ivy (jssarma)
Modified: hadoop/hive/trunk/ant/build.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ant/build.xml?rev=801012&r1=801011&r2=801012&view=diff
==============================================================================
--- hadoop/hive/trunk/ant/build.xml (original)
+++ hadoop/hive/trunk/ant/build.xml Tue Aug 4 23:10:53 2009
@@ -27,6 +27,22 @@
<property name="src.dir" location="${basedir}/src"/>
<import file="../build-common.xml"/>
+ <target name="init" depends="create-dirs"/>
+
+ <target name="compile" depends="init">
+ <echo message="Compiling: ${ant.project.name}"/>
+ <javac
+ encoding="${build.encoding}"
+ srcdir="${src.dir}"
+ includes="**/*.java"
+ destdir="${build.classes}"
+ debug="${javac.debug}"
+ deprecation="${javac.deprecation}">
+ <compilerarg line="${javac.args} ${javac.args.warnings}" />
+ <classpath refid="classpath"/>
+ </javac>
+ </target>
+
<target name="jar" depends="compile">
<copy file="${src.dir}/org/apache/hadoop/hive/ant/antlib.xml"
todir="${build.dir}/classes/org/apache/hadoop/hive/ant"/>
Modified: hadoop/hive/trunk/bin/ext/cli.sh
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/bin/ext/cli.sh?rev=801012&r1=801011&r2=801012&view=diff
==============================================================================
--- hadoop/hive/trunk/bin/ext/cli.sh (original)
+++ hadoop/hive/trunk/bin/ext/cli.sh Tue Aug 4 23:10:53 2009
@@ -9,8 +9,19 @@
echo "Missing Hive CLI Jar"
exit 3;
fi
+
+ for f in ${HADOOP_HOME}/hadoop*core.jar ${HADOOP_HOME}/lib/hadoop*core.jar; do
+ if [[ ! -f $f ]]; then
+ continue;
+ fi
+ if [[ $f == *17* ]] || [[ $f == *18* ]] || [[ $f == *19* ]]; then
+ exec $HADOOP jar $AUX_JARS_CMD_LINE ${HIVE_LIB}/hive_cli.jar $CLASS $HIVE_OPTS "$@"
+ else
+ # hadoop 20 or newer - skip the aux_jars option. picked up from hiveconf
+ exec $HADOOP jar ${HIVE_LIB}/hive_cli.jar $CLASS $HIVE_OPTS "$@"
+ fi
+ done
- exec $HADOOP jar $AUX_JARS_CMD_LINE ${HIVE_LIB}/hive_cli.jar $CLASS $HIVE_OPTS "$@"
}
cli_help () {
Modified: hadoop/hive/trunk/build-common.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/build-common.xml?rev=801012&r1=801011&r2=801012&view=diff
==============================================================================
--- hadoop/hive/trunk/build-common.xml (original)
+++ hadoop/hive/trunk/build-common.xml Tue Aug 4 23:10:53 2009
@@ -79,9 +79,9 @@
<ivy:retrieve pattern="${build.dir.hadoop}/[artifact]-[revision].[ext]"/>
</target>
- <available property="hadoopcore.install.done" file="${build.dir.hadoop}/hadoop-${hadoop.version}.installed"/>
+ <available property="hadoopcore.${hadoop.version}.install.done" file="${build.dir.hadoop}/hadoop-${hadoop.version}.installed"/>
- <target name="install-hadoopcore-internal" depends="resolve" unless="hadoopcore.install.done">
+ <target name="install-hadoopcore-internal" depends="resolve" unless="hadoopcore.${hadoop.version}.install.done">
<untar src="${build.dir.hadoop}/hadoop-${hadoop.version}.tar.gz" dest="${build.dir.hadoop}" compression="gzip"/>
<chmod file="${hadoop.root}/bin/hadoop" perm="+x"/>
<touch file="${build.dir.hadoop}/hadoop-${hadoop.version}.installed"/>
@@ -116,11 +116,12 @@
<pathelement location="${build.dir.hive}/metastore/classes"/>
<pathelement location="${build.dir.hive}/ql/classes"/>
<pathelement location="${build.dir.hive}/cli/classes"/>
+ <pathelement location="${build.dir.hive}/shims/classes"/>
<fileset dir="${basedir}" includes="lib/*.jar"/>
<path refid="common-classpath"/>
</path>
- <target name="init">
+ <target name="create-dirs">
<mkdir dir="${build.dir.hive}"/>
<mkdir dir="${build.dir}"/>
<mkdir dir="${build.classes}"/>
@@ -131,6 +132,8 @@
<mkdir dir="${test.build.classes}"/>
</target>
+ <target name="init" depends="create-dirs, deploy-ant-tasks"/>
+
<target name="test-init">
<mkdir dir="${test.data.dir}"/>
<mkdir dir="${test.log.dir}/clientpositive"/>
@@ -144,7 +147,9 @@
</copy>
</target>
- <target name="compile" depends="init, install-hadoopcore">
+ <target name="setup"/>
+
+ <target name="compile" depends="init, install-hadoopcore, setup">
<echo message="Compiling: ${ant.project.name}"/>
<javac
encoding="${build.encoding}"
@@ -245,7 +250,7 @@
<!-- target to deploy anttasks -->
- <target name="compile-ant-tasks" depends="init">
+ <target name="compile-ant-tasks">
<subant target="compile">
<fileset dir=".." includes="ant/build.xml"/>
</subant>
@@ -255,6 +260,10 @@
<subant target="jar">
<fileset dir=".." includes="ant/build.xml"/>
</subant>
+
+ <taskdef name="getversionpref" classname="org.apache.hadoop.hive.ant.GetVersionPref"
+ classpath="${build.dir.hive}/anttasks/hive_anttasks.jar"/>
+
</target>
<target name="gen-test"/>
Modified: hadoop/hive/trunk/build.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/build.xml?rev=801012&r1=801011&r2=801012&view=diff
==============================================================================
--- hadoop/hive/trunk/build.xml (original)
+++ hadoop/hive/trunk/build.xml Tue Aug 4 23:10:53 2009
@@ -62,7 +62,7 @@
<attribute name="target"/>
<sequential>
<subant target="@{target}">
- <filelist dir="." files="common/build.xml,serde/build.xml,metastore/build.xml,ql/build.xml,cli/build.xml,contrib/build.xml,service/build.xml,jdbc/build.xml,hwi/build.xml,ant/build.xml"/>
+ <filelist dir="." files="ant/build.xml,shims/build.xml,common/build.xml,serde/build.xml,metastore/build.xml,ql/build.xml,cli/build.xml,contrib/build.xml,service/build.xml,jdbc/build.xml,hwi/build.xml,ant/build.xml"/>
</subant>
</sequential>
</macrodef>
@@ -71,7 +71,7 @@
<attribute name="target"/>
<sequential>
<subant target="@{target}">
- <filelist dir="." files="common/build.xml,serde/build.xml,metastore/build.xml,ql/build.xml,cli/build.xml,contrib/build.xml,service/build.xml,jdbc/build.xml,hwi/build.xml"/>
+ <filelist dir="." files="shims/build.xml,common/build.xml,serde/build.xml,metastore/build.xml,ql/build.xml,cli/build.xml,contrib/build.xml,service/build.xml,jdbc/build.xml,hwi/build.xml"/>
</subant>
</sequential>
</macrodef>
@@ -217,6 +217,11 @@
</condition>
<echo message="Using hadoop version ${hadoop.version}"/>
+ <taskdef name="getversionpref" classname="org.apache.hadoop.hive.ant.GetVersionPref"
+ classpath="${build.dir.hive}/anttasks/hive_anttasks.jar"/>
+
+ <getversionpref property="hadoop.version.prefix" input="${hadoop.version}"/>
+
<pathconvert property="eclipse.project">
<path path="${basedir}"/>
<regexpmapper from="^.*/([^/]+)$$" to="\1" handledirsep="yes"/>
@@ -229,6 +234,7 @@
<filterset>
<filter token="PROJECT" value="${eclipse.project}"/>
<filter token="HADOOPVER" value="${hadoop.version}"/>
+ <filter token="HADOOPVERPREF" value="${hadoop.version.prefix}"/>
</filterset>
</copy>
<move todir="." includeemptydirs="false">
Modified: hadoop/hive/trunk/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java?rev=801012&r1=801011&r2=801012&view=diff
==============================================================================
--- hadoop/hive/trunk/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java (original)
+++ hadoop/hive/trunk/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java Tue Aug 4 23:10:53 2009
@@ -37,6 +37,7 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hive.shims.ShimLoader;
public class CliDriver {
@@ -191,7 +192,7 @@
return (processLine(qsb.toString()));
}
- public static void main(String[] args) throws IOException {
+ public static void main(String[] args) throws Exception {
OptionsProcessor oproc = new OptionsProcessor();
if(! oproc.process_stage1(args)) {
@@ -222,6 +223,18 @@
conf.set((String) item.getKey(), (String) item.getValue());
}
+ if(!ShimLoader.getHadoopShims().usesJobShell()) {
+ // hadoop-20 and above - we need to augment classpath using hiveconf components
+ // see also: code in ExecDriver.java
+ ClassLoader loader = conf.getClassLoader();
+ String auxJars = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEAUXJARS);
+ if (StringUtils.isNotBlank(auxJars)) {
+ loader = Utilities.addToClassPath(loader, StringUtils.split(auxJars, ","));
+ }
+ conf.setClassLoader(loader);
+ Thread.currentThread().setContextClassLoader(loader);
+ }
+
SessionState.start(ss);
CliDriver cli = new CliDriver ();
Modified: hadoop/hive/trunk/contrib/src/java/org/apache/hadoop/hive/contrib/fileformat/base64/Base64TextInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/contrib/src/java/org/apache/hadoop/hive/contrib/fileformat/base64/Base64TextInputFormat.java?rev=801012&r1=801011&r2=801012&view=diff
==============================================================================
--- hadoop/hive/trunk/contrib/src/java/org/apache/hadoop/hive/contrib/fileformat/base64/Base64TextInputFormat.java (original)
+++ hadoop/hive/trunk/contrib/src/java/org/apache/hadoop/hive/contrib/fileformat/base64/Base64TextInputFormat.java Tue Aug 4 23:10:53 2009
@@ -20,9 +20,9 @@
import java.io.IOException;
import java.io.UnsupportedEncodingException;
-import java.lang.reflect.Method;
import java.util.Arrays;
+import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
@@ -169,13 +169,7 @@
// Cannot put @Override here because hadoop 0.18+ removed this method.
public void validateInput(JobConf job) throws IOException {
- try {
- Method validateInput = format.getClass().getDeclaredMethod("validateInput", job.getClass());
- validateInput.setAccessible(true);
- validateInput.invoke(format, job);
- } catch (Exception e) {
- // Ignore this exception since validateInput is removed from hadoop in 0.18+.
- }
+ ShimLoader.getHadoopShims().inputFormatValidateInput(format, job);
}
}
Modified: hadoop/hive/trunk/data/conf/hive-log4j.properties
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/data/conf/hive-log4j.properties?rev=801012&r1=801011&r2=801012&view=diff
==============================================================================
--- hadoop/hive/trunk/data/conf/hive-log4j.properties (original)
+++ hadoop/hive/trunk/data/conf/hive-log4j.properties Tue Aug 4 23:10:53 2009
@@ -58,4 +58,5 @@
log4j.category.JPOX.Query=ERROR,DRFA
log4j.category.JPOX.General=ERROR,DRFA
log4j.category.JPOX.Enhancer=ERROR,DRFA
+log4j.logger.org.apache.hadoop.conf.Configuration=ERROR,DRFA
Modified: hadoop/hive/trunk/eclipse-templates/.classpath
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/eclipse-templates/.classpath?rev=801012&r1=801011&r2=801012&view=diff
==============================================================================
--- hadoop/hive/trunk/eclipse-templates/.classpath (original)
+++ hadoop/hive/trunk/eclipse-templates/.classpath Tue Aug 4 23:10:53 2009
@@ -44,6 +44,7 @@
<classpathentry kind="src" path="service/src/test"/>
<classpathentry kind="src" path="jdbc/src/java"/>
<classpathentry kind="src" path="jdbc/src/test"/>
+ <classpathentry kind="src" path="shims/src/@HADOOPVERPREF@/java"/>
<classpathentry kind="src" path="hwi/src/java"/>
<classpathentry kind="src" path="hwi/src/test"/>
<classpathentry kind="output" path="build/eclipse-classes"/>
Modified: hadoop/hive/trunk/hwi/src/java/org/apache/hadoop/hive/hwi/HWIServer.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/hwi/src/java/org/apache/hadoop/hive/hwi/HWIServer.java?rev=801012&r1=801011&r2=801012&view=diff
==============================================================================
--- hadoop/hive/trunk/hwi/src/java/org/apache/hadoop/hive/hwi/HWIServer.java (original)
+++ hadoop/hive/trunk/hwi/src/java/org/apache/hadoop/hive/hwi/HWIServer.java Tue Aug 4 23:10:53 2009
@@ -6,9 +6,11 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.mortbay.http.SocketListener;
import org.mortbay.jetty.Server;
-import org.mortbay.jetty.servlet.WebApplicationContext;
+
+import org.apache.hadoop.hive.shims.JettyShims;
+import org.apache.hadoop.hive.shims.ShimLoader;
+
/**
* This is the entry point for HWI. A web server is invoked in the same manner as the hive CLI.
* Rather then opening a command line session a web server is started and a web application to work with
@@ -17,8 +19,7 @@
public class HWIServer {
protected static final Log l4j = LogFactory.getLog( HWIServer.class.getName() );
- private org.mortbay.jetty.Server webServer;
- private SocketListener listener;
+ private JettyShims.Server webServer;
private String [] args;
/**
@@ -36,8 +37,6 @@
*/
public void start() throws IOException {
- webServer = new org.mortbay.jetty.Server();
- listener = new SocketListener();
HiveConf conf = new HiveConf(this.getClass());
String listen = null;
@@ -54,21 +53,16 @@
l4j.warn("hive.hwi.listen.port was not specified defaulting to 9999");
port=9999;
}
-
- listener.setPort(port);
- listener.setHost(listen);
-
- webServer.addListener(listener);
-
- WebApplicationContext wac = new WebApplicationContext();
- wac.setContextPath("/hwi");
+
+
String hwiWAR = conf.getVar(HiveConf.ConfVars.HIVEHWIWARFILE);
if (! new File (hwiWAR).exists() ){
l4j.fatal("HWI WAR file not found at "+ hwiWAR );
}
- wac.setWAR(hwiWAR);
+ webServer = ShimLoader.getJettyShims().startServer(listen, port);
+ webServer.addWar(hwiWAR, "/hwi");
/*The command line args may be used by multiple components. Rather by setting
* these as a system property we avoid having to specifically pass them
@@ -79,7 +73,6 @@
}
System.setProperty("hwi-args", sb.toString());
- webServer.addContext(wac);
try {
while (true) {
try {
@@ -117,9 +110,9 @@
/**
* Shut down the running HWI Server
- * @throws InterruptedException Running Thread.stop() can and probably will throw this
+ * @throws Exception Running Thread.stop() can and probably will throw this
*/
- public void stop() throws InterruptedException {
+ public void stop() throws Exception {
l4j.info("HWI is shutting down");
webServer.stop();
}
Modified: hadoop/hive/trunk/ql/build.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/build.xml?rev=801012&r1=801011&r2=801012&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/build.xml (original)
+++ hadoop/hive/trunk/ql/build.xml Tue Aug 4 23:10:53 2009
@@ -152,6 +152,7 @@
<fileset dir="${build.dir.hive}/thrift/classes" includes="**/*.class"/>
<fileset dir="${build.dir.hive}/commons-lang/classes" includes="**/StringUtils.class,**/WordUtils.class"/>
<fileset dir="${build.dir.hive}/json/classes" includes="**/*.class"/>
+ <fileset dir="${build.dir.hive}/shims/classes" includes="**/*.class"/>
</jar>
</target>
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java?rev=801012&r1=801011&r2=801012&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java Tue Aug 4 23:10:53 2009
@@ -41,6 +41,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.hive.ql.plan.mapredWork;
import org.apache.hadoop.hive.ql.plan.exprNodeDesc;
import org.apache.hadoop.hive.ql.plan.partitionDesc;
@@ -91,12 +92,7 @@
private void initializeFiles(String prop, String files) {
if (files != null && files.length() > 0) {
job.set(prop, files);
-
- // workaround for hadoop-17 - jobclient only looks at commandlineconfig
- Configuration commandConf = JobClient.getCommandLineConfig();
- if (commandConf != null) {
- commandConf.set(prop, files);
- }
+ ShimLoader.getHadoopShims().setTmpFiles(prop, files);
}
}
@@ -552,6 +548,7 @@
String auxJars = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEAUXJARS);
String addedJars = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEADDEDJARS);
try {
+ // see also - code in CliDriver.java
ClassLoader loader = conf.getClassLoader();
if (StringUtils.isNotBlank(auxJars)) {
loader = Utilities.addToClassPath(loader, StringUtils.split(auxJars, ","));
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java?rev=801012&r1=801011&r2=801012&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java Tue Aug 4 23:10:53 2009
@@ -20,7 +20,6 @@
import java.io.IOException;
import java.io.Serializable;
-import java.lang.reflect.Method;
import java.util.Properties;
import org.apache.hadoop.conf.Configuration;
@@ -33,6 +32,7 @@
import org.apache.hadoop.hive.ql.plan.tableDesc;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.Serializer;
+import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Writable;
@@ -117,12 +117,7 @@
outWriter = getRecordWriter(jc, hiveOutputFormat, outputClass, isCompressed, tableInfo.getProperties(), outPath);
// in recent hadoop versions, use deleteOnExit to clean tmp files.
- try {
- Method deleteOnExit = FileSystem.class.getDeclaredMethod("deleteOnExit", new Class [] {Path.class});
- deleteOnExit.setAccessible(true);
- deleteOnExit.invoke(fs, outPath);
- autoDelete = true;
- } catch (Exception e) {}
+ autoDelete = ShimLoader.getHadoopShims().fileSystemDeleteOnExit(fs, outPath);
initializeChildren(hconf);
} catch (HiveException e) {
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapRedTask.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapRedTask.java?rev=801012&r1=801011&r2=801012&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapRedTask.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapRedTask.java Tue Aug 4 23:10:53 2009
@@ -30,6 +30,7 @@
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.commons.lang.StringUtils;
@@ -83,9 +84,16 @@
String isSilent = "true".equalsIgnoreCase(System.getProperty("test.silent"))
? "-silent" : "";
- String cmdLine = hadoopExec + " jar " + libJarsOption + " " + hiveJar
- + " org.apache.hadoop.hive.ql.exec.ExecDriver -plan "
- + planFile.toString() + " " + isSilent + " " + hiveConfArgs;
+
+ String jarCmd;
+ if(ShimLoader.getHadoopShims().usesJobShell()) {
+ jarCmd = libJarsOption + hiveJar + " " + ExecDriver.class.getName();
+ } else {
+ jarCmd = hiveJar + " " + ExecDriver.class.getName() + libJarsOption;
+ }
+
+ String cmdLine = hadoopExec + " jar " + jarCmd +
+ " -plan " + planFile.toString() + " " + isSilent + " " + hiveConfArgs;
String files = ExecDriver.getResourceFiles(conf, SessionState.ResourceType.FILE);
if(!files.isEmpty()) {
@@ -102,7 +110,7 @@
executor = Runtime.getRuntime().exec(cmdLine);
// user specified the memory - only applicable for local mode
else {
- Map<String, String> variables = System.getenv();
+ Map<String, String> variables = System.getenv();
String[] env = new String[variables.size() + 1];
int pos = 0;
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java?rev=801012&r1=801011&r2=801012&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java Tue Aug 4 23:10:53 2009
@@ -22,7 +22,6 @@
import java.io.DataOutput;
import java.io.IOException;
import java.io.Serializable;
-import java.lang.reflect.Method;
import java.net.URLClassLoader;
import java.util.ArrayList;
import java.util.Arrays;
@@ -42,6 +41,7 @@
import org.apache.hadoop.hive.ql.plan.mapredWork;
import org.apache.hadoop.hive.ql.plan.tableDesc;
import org.apache.hadoop.hive.ql.plan.partitionDesc;
+import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapred.FileInputFormat;
@@ -291,13 +291,7 @@
FileInputFormat.setInputPaths(newjob, dir);
newjob.setInputFormat(inputFormat.getClass());
- try {
- Method validateInput = inputFormat.getClass().getDeclaredMethod("validateInput", newjob.getClass());
- validateInput.setAccessible(true);
- validateInput.invoke(inputFormat, newjob);
- } catch (Exception e) {
- // Ignore this exception since validateInput is removed from hadoop in 0.18+.
- }
+ ShimLoader.getHadoopShims().inputFormatValidateInput(inputFormat, newjob);
}
}
Modified: hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java?rev=801012&r1=801011&r2=801012&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java (original)
+++ hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java Tue Aug 4 23:10:53 2009
@@ -34,8 +34,6 @@
import java.util.TreeMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import java.lang.reflect.Method;
-import java.lang.reflect.Constructor;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -60,6 +58,8 @@
import org.apache.hadoop.hive.serde.Constants;
import org.apache.hadoop.hive.serde2.thrift.ThriftDeserializer;
import org.apache.hadoop.hive.serde2.thrift.test.Complex;
+import org.apache.hadoop.hive.shims.HadoopShims;
+import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.mapred.SequenceFileInputFormat;
import org.apache.hadoop.mapred.SequenceFileOutputFormat;
import org.apache.hadoop.mapred.TextInputFormat;
@@ -87,9 +87,8 @@
private boolean overWrite;
private CliDriver cliDriver;
private MiniMRCluster mr = null;
- private Object dfs = null;
+ private HadoopShims.MiniDFSShim dfs = null;
private boolean miniMr = false;
- private Class<?> dfsClass = null;
public boolean deleteDirectory(File path) {
if (path.exists()) {
@@ -175,30 +174,8 @@
qMap = new TreeMap<String, String>();
if (miniMr) {
- dfsClass = null;
-
- // The path for MiniDFSCluster has changed, so look in both 17 and 19
- // In hadoop 17, the path is org.apache.hadoop.dfs.MiniDFSCluster, whereas
- // it is org.apache.hadoop.hdfs.MiniDFSCluster in hadoop 19. Due to this anamonly,
- // use reflection to invoke the methods.
- try {
- dfsClass = Class.forName("org.apache.hadoop.dfs.MiniDFSCluster");
- } catch (ClassNotFoundException e) {
- dfsClass = null;
- }
-
- if (dfsClass == null) {
- dfsClass = Class.forName("org.apache.hadoop.hdfs.MiniDFSCluster");
- }
-
- Constructor<?> dfsCons =
- dfsClass.getDeclaredConstructor(new Class<?>[] {Configuration.class, Integer.TYPE,
- Boolean.TYPE, (new String[] {}).getClass()});
-
- dfs = dfsCons.newInstance(conf, 4, true, null);
- Method m = dfsClass.getDeclaredMethod("getFileSystem", new Class[]{});
- FileSystem fs = (FileSystem)m.invoke(dfs, new Object[] {});
-
+ dfs = ShimLoader.getHadoopShims().getMiniDfs(conf, 4, true, null);
+ FileSystem fs = dfs.getFileSystem();
mr = new MiniMRCluster(4, fs.getUri().toString(), 1);
// hive.metastore.warehouse.dir needs to be set relative to the jobtracker
@@ -226,10 +203,8 @@
cleanUp();
if (dfs != null) {
- Method m = dfsClass.getDeclaredMethod("shutdown", new Class[]{});
- m.invoke(dfs, new Object[]{});
+ dfs.shutdown();
dfs = null;
- dfsClass = null;
}
if (mr != null) {
Added: hadoop/hive/trunk/shims/build.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/shims/build.xml?rev=801012&view=auto
==============================================================================
--- hadoop/hive/trunk/shims/build.xml (added)
+++ hadoop/hive/trunk/shims/build.xml Tue Aug 4 23:10:53 2009
@@ -0,0 +1,73 @@
+<?xml version="1.0"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+
+<!--
+Before you can run these subtargets directly, you need
+to call at top-level: ant deploy-contrib compile-core-test
+-->
+<project name="shims" default="jar">
+ <import file="../build-common.xml"/>
+
+ <path id="classpath">
+ <pathelement location="${hadoop.jar}"/>
+ <pathelement location="${hadoop.test.jar}"/>
+ <fileset dir="${hadoop.root}/lib">
+ <include name="**/*.jar" />
+ <exclude name="**/excluded/" />
+ </fileset>
+ </path>
+
+ <target name="build_shims" description="Build shims against a particular hadoop version" depends="install-hadoopcore-internal, resolve">
+ <getversionpref property="hadoop.version.prefix" input="${hadoop.version}" />
+ <echo message="Compiling shims against hadoop ${hadoop.version}"/>
+ <javac
+ encoding="${build.encoding}"
+ includes="**/*.java"
+ destdir="${build.classes}"
+ debug="${javac.debug}"
+ deprecation="${javac.deprecation}">
+ <compilerarg line="${javac.args} ${javac.args.warnings}" />
+ <classpath refid="classpath"/>
+ <src path="${basedir}/src/${hadoop.version.prefix}/java" />
+ <src path="${basedir}/src/common/java" />
+ </javac>
+ </target>
+
+ <target name="compile" depends="init, install-hadoopcore, setup">
+ <antcall target="build_shims" inheritRefs="false" inheritAll="false">
+ <param name="hadoop.version" value="0.17.2.1" />
+ </antcall>
+ <antcall target="build_shims" inheritRefs="false" inheritAll="false">
+ <param name="hadoop.version" value="0.18.3" />
+ </antcall>
+ <antcall target="build_shims" inheritRefs="false" inheritAll="false">
+ <param name="hadoop.version" value="0.19.0" />
+ </antcall>
+ <antcall target="build_shims" inheritRefs="false" inheritAll="false">
+ <param name="hadoop.version" value="0.20.0" />
+ </antcall>
+ </target>
+
+
+ <target name="test">
+ <echo message="Nothing to do!"/>
+ </target>
+
+</project>
Added: hadoop/hive/trunk/shims/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/shims/ivy.xml?rev=801012&view=auto
==============================================================================
--- hadoop/hive/trunk/shims/ivy.xml (added)
+++ hadoop/hive/trunk/shims/ivy.xml Tue Aug 4 23:10:53 2009
@@ -0,0 +1,18 @@
+<ivy-module version="2.0">
+ <info organisation="org.apache.hadoop.hive" module="shims"/>
+ <dependencies>
+ <dependency org="hadoop" name="core" rev="0.17.2.1">
+ <artifact name="hadoop" type="source" ext="tar.gz"/>
+ </dependency>
+ <dependency org="hadoop" name="core" rev="0.18.3">
+ <artifact name="hadoop" type="source" ext="tar.gz"/>
+ </dependency>
+ <dependency org="hadoop" name="core" rev="0.19.0">
+ <artifact name="hadoop" type="source" ext="tar.gz"/>
+ </dependency>
+ <dependency org="hadoop" name="core" rev="0.20.0">
+ <artifact name="hadoop" type="source" ext="tar.gz"/>
+ </dependency>
+ <conflict manager="all" />
+ </dependencies>
+</ivy-module>
Added: hadoop/hive/trunk/shims/src/0.17/java/org/apache/hadoop/hive/shims/Hadoop17Shims.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/shims/src/0.17/java/org/apache/hadoop/hive/shims/Hadoop17Shims.java?rev=801012&view=auto
==============================================================================
--- hadoop/hive/trunk/shims/src/0.17/java/org/apache/hadoop/hive/shims/Hadoop17Shims.java (added)
+++ hadoop/hive/trunk/shims/src/0.17/java/org/apache/hadoop/hive/shims/Hadoop17Shims.java Tue Aug 4 23:10:53 2009
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.shims;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.dfs.MiniDFSCluster;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import java.io.IOException;
+
+/**
+ * Implemention of shims against Hadoop 0.17.0
+ */
+public class Hadoop17Shims implements HadoopShims {
+ public boolean usesJobShell() {
+ return true;
+ }
+
+ public boolean fileSystemDeleteOnExit(FileSystem fs, Path path)
+ throws IOException {
+ return false;
+ }
+
+ public void inputFormatValidateInput(InputFormat fmt, JobConf conf)
+ throws IOException {
+ fmt.validateInput(conf);
+ }
+
+ /**
+ * workaround for hadoop-17 - jobclient only looks at commandlineconfig
+ */
+ public void setTmpFiles(String prop, String files) {
+ Configuration conf = JobClient.getCommandLineConfig();
+ if (conf != null) {
+ conf.set(prop, files);
+ }
+ }
+
+ public HadoopShims.MiniDFSShim getMiniDfs(Configuration conf,
+ int numDataNodes,
+ boolean format,
+ String[] racks) throws IOException {
+ return new MiniDFSShim(new MiniDFSCluster(conf, numDataNodes, format, racks));
+ }
+
+ public class MiniDFSShim implements HadoopShims.MiniDFSShim {
+ private MiniDFSCluster cluster;
+ public MiniDFSShim(MiniDFSCluster cluster) {
+ this.cluster = cluster;
+ }
+
+ public FileSystem getFileSystem() throws IOException {
+ return cluster.getFileSystem();
+ }
+
+ public void shutdown() {
+ cluster.shutdown();
+ }
+ }
+}
Added: hadoop/hive/trunk/shims/src/0.17/java/org/apache/hadoop/hive/shims/Jetty17Shims.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/shims/src/0.17/java/org/apache/hadoop/hive/shims/Jetty17Shims.java?rev=801012&view=auto
==============================================================================
--- hadoop/hive/trunk/shims/src/0.17/java/org/apache/hadoop/hive/shims/Jetty17Shims.java (added)
+++ hadoop/hive/trunk/shims/src/0.17/java/org/apache/hadoop/hive/shims/Jetty17Shims.java Tue Aug 4 23:10:53 2009
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.shims;
+
+import org.mortbay.http.SocketListener;
+import org.mortbay.jetty.servlet.WebApplicationContext;
+
+import java.io.IOException;
+
+public class Jetty17Shims implements JettyShims {
+ public Server startServer(String listen, int port) throws IOException {
+ Server s = new Server();
+ s.setupListenerHostPort(listen, port);
+ return s;
+ }
+
+ private static class Server extends org.mortbay.jetty.Server implements JettyShims.Server {
+ public void addWar(String war, String contextPath) {
+ WebApplicationContext wac = new WebApplicationContext();
+ wac.setContextPath(contextPath);
+ wac.setWAR(war);
+ addContext(wac);
+ }
+
+ public void setupListenerHostPort(String listen, int port)
+ throws IOException {
+
+ SocketListener listener = new SocketListener();
+ listener.setPort(port);
+ listener.setHost(listen);
+ this.addListener(listener);
+ }
+ }
+}
\ No newline at end of file
Added: hadoop/hive/trunk/shims/src/0.18/java/org/apache/hadoop/hive/shims/Hadoop18Shims.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/shims/src/0.18/java/org/apache/hadoop/hive/shims/Hadoop18Shims.java?rev=801012&view=auto
==============================================================================
--- hadoop/hive/trunk/shims/src/0.18/java/org/apache/hadoop/hive/shims/Hadoop18Shims.java (added)
+++ hadoop/hive/trunk/shims/src/0.18/java/org/apache/hadoop/hive/shims/Hadoop18Shims.java Tue Aug 4 23:10:53 2009
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.shims;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.dfs.MiniDFSCluster;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import java.io.IOException;
+
+/**
+ * Implemention of shims against Hadoop 0.18.0
+ */
+public class Hadoop18Shims implements HadoopShims {
+ public boolean usesJobShell() {
+ return true;
+ }
+
+ public boolean fileSystemDeleteOnExit(FileSystem fs, Path path)
+ throws IOException {
+
+ return fs.deleteOnExit(path);
+ }
+
+ public void inputFormatValidateInput(InputFormat fmt, JobConf conf)
+ throws IOException {
+ // gone in 0.18+
+ }
+
+ /**
+ * workaround for hadoop-17 - jobclient only looks at commandlineconfig
+ */
+ public void setTmpFiles(String prop, String files) {
+ Configuration conf = JobClient.getCommandLineConfig();
+ if (conf != null) {
+ conf.set(prop, files);
+ }
+ }
+
+ public HadoopShims.MiniDFSShim getMiniDfs(Configuration conf,
+ int numDataNodes,
+ boolean format,
+ String[] racks) throws IOException {
+ return new MiniDFSShim(new MiniDFSCluster(conf, numDataNodes, format, racks));
+ }
+
+ public class MiniDFSShim implements HadoopShims.MiniDFSShim {
+ private MiniDFSCluster cluster;
+ public MiniDFSShim(MiniDFSCluster cluster) {
+ this.cluster = cluster;
+ }
+
+ public FileSystem getFileSystem() throws IOException {
+ return cluster.getFileSystem();
+ }
+
+ public void shutdown() {
+ cluster.shutdown();
+ }
+ }
+}
\ No newline at end of file
Added: hadoop/hive/trunk/shims/src/0.18/java/org/apache/hadoop/hive/shims/Jetty18Shims.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/shims/src/0.18/java/org/apache/hadoop/hive/shims/Jetty18Shims.java?rev=801012&view=auto
==============================================================================
--- hadoop/hive/trunk/shims/src/0.18/java/org/apache/hadoop/hive/shims/Jetty18Shims.java (added)
+++ hadoop/hive/trunk/shims/src/0.18/java/org/apache/hadoop/hive/shims/Jetty18Shims.java Tue Aug 4 23:10:53 2009
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.shims;
+
+import org.mortbay.http.SocketListener;
+import org.mortbay.jetty.servlet.WebApplicationContext;
+
+import java.io.IOException;
+
+public class Jetty18Shims implements JettyShims {
+ public Server startServer(String listen, int port) throws IOException {
+ Server s = new Server();
+ s.setupListenerHostPort(listen, port);
+ return s;
+ }
+
+ private static class Server extends org.mortbay.jetty.Server implements JettyShims.Server {
+ public void addWar(String war, String contextPath) {
+ WebApplicationContext wac = new WebApplicationContext();
+ wac.setContextPath(contextPath);
+ wac.setWAR(war);
+ addContext(wac);
+ }
+
+ public void setupListenerHostPort(String listen, int port)
+ throws IOException {
+
+ SocketListener listener = new SocketListener();
+ listener.setPort(port);
+ listener.setHost(listen);
+ this.addListener(listener);
+ }
+ }
+}
\ No newline at end of file
Added: hadoop/hive/trunk/shims/src/0.19/java/org/apache/hadoop/hive/shims/Hadoop19Shims.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/shims/src/0.19/java/org/apache/hadoop/hive/shims/Hadoop19Shims.java?rev=801012&view=auto
==============================================================================
--- hadoop/hive/trunk/shims/src/0.19/java/org/apache/hadoop/hive/shims/Hadoop19Shims.java (added)
+++ hadoop/hive/trunk/shims/src/0.19/java/org/apache/hadoop/hive/shims/Hadoop19Shims.java Tue Aug 4 23:10:53 2009
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.shims;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import java.io.IOException;
+
+/**
+ * Implemention of shims against Hadoop 0.19.0
+ */
+public class Hadoop19Shims implements HadoopShims {
+ public boolean usesJobShell() {
+ return true;
+ }
+
+ public boolean fileSystemDeleteOnExit(FileSystem fs, Path path)
+ throws IOException {
+
+ return fs.deleteOnExit(path);
+ }
+
+ public void inputFormatValidateInput(InputFormat fmt, JobConf conf)
+ throws IOException {
+ // gone in 0.18+
+ }
+
+ /**
+ * workaround for hadoop-17 - jobclient only looks at commandlineconfig
+ */
+ public void setTmpFiles(String prop, String files) {
+ Configuration conf = JobClient.getCommandLineConfig();
+ if (conf != null) {
+ conf.set(prop, files);
+ }
+ }
+
+ public HadoopShims.MiniDFSShim getMiniDfs(Configuration conf,
+ int numDataNodes,
+ boolean format,
+ String[] racks) throws IOException {
+ return new MiniDFSShim(new MiniDFSCluster(conf, numDataNodes, format, racks));
+ }
+
+ public class MiniDFSShim implements HadoopShims.MiniDFSShim {
+ private MiniDFSCluster cluster;
+ public MiniDFSShim(MiniDFSCluster cluster) {
+ this.cluster = cluster;
+ }
+
+ public FileSystem getFileSystem() throws IOException {
+ return cluster.getFileSystem();
+ }
+
+ public void shutdown() {
+ cluster.shutdown();
+ }
+ }
+}
\ No newline at end of file
Added: hadoop/hive/trunk/shims/src/0.19/java/org/apache/hadoop/hive/shims/Jetty19Shims.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/shims/src/0.19/java/org/apache/hadoop/hive/shims/Jetty19Shims.java?rev=801012&view=auto
==============================================================================
--- hadoop/hive/trunk/shims/src/0.19/java/org/apache/hadoop/hive/shims/Jetty19Shims.java (added)
+++ hadoop/hive/trunk/shims/src/0.19/java/org/apache/hadoop/hive/shims/Jetty19Shims.java Tue Aug 4 23:10:53 2009
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.shims;
+
+import org.mortbay.http.SocketListener;
+import org.mortbay.jetty.servlet.WebApplicationContext;
+
+import java.io.IOException;
+
+public class Jetty19Shims implements JettyShims {
+ public Server startServer(String listen, int port) throws IOException {
+ Server s = new Server();
+ s.setupListenerHostPort(listen, port);
+ return s;
+ }
+
+ private static class Server extends org.mortbay.jetty.Server implements JettyShims.Server {
+ public void addWar(String war, String contextPath) {
+ WebApplicationContext wac = new WebApplicationContext();
+ wac.setContextPath(contextPath);
+ wac.setWAR(war);
+ addContext(wac);
+ }
+
+ public void setupListenerHostPort(String listen, int port)
+ throws IOException {
+
+ SocketListener listener = new SocketListener();
+ listener.setPort(port);
+ listener.setHost(listen);
+ this.addListener(listener);
+ }
+ }
+}
\ No newline at end of file
Added: hadoop/hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java?rev=801012&view=auto
==============================================================================
--- hadoop/hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java (added)
+++ hadoop/hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java Tue Aug 4 23:10:53 2009
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.shims;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import java.io.IOException;
+
+/**
+ * Implemention of shims against Hadoop 0.20.0
+ */
+public class Hadoop20Shims implements HadoopShims {
+ public boolean usesJobShell() {
+ return false;
+ }
+
+ public boolean fileSystemDeleteOnExit(FileSystem fs, Path path)
+ throws IOException {
+
+ return fs.deleteOnExit(path);
+ }
+
+ public void inputFormatValidateInput(InputFormat fmt, JobConf conf)
+ throws IOException {
+ // gone in 0.18+
+ }
+
+ /**
+ * workaround for hadoop-17 - jobclient only looks at commandlineconfig
+ */
+ public void setTmpFiles(String prop, String files) {
+ // gone in 20+
+ }
+
+ public HadoopShims.MiniDFSShim getMiniDfs(Configuration conf,
+ int numDataNodes,
+ boolean format,
+ String[] racks) throws IOException {
+ return new MiniDFSShim(new MiniDFSCluster(conf, numDataNodes, format, racks));
+ }
+
+ public class MiniDFSShim implements HadoopShims.MiniDFSShim {
+ private MiniDFSCluster cluster;
+ public MiniDFSShim(MiniDFSCluster cluster) {
+ this.cluster = cluster;
+ }
+
+ public FileSystem getFileSystem() throws IOException {
+ return cluster.getFileSystem();
+ }
+
+ public void shutdown() {
+ cluster.shutdown();
+ }
+ }
+}
\ No newline at end of file
Added: hadoop/hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Jetty20Shims.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Jetty20Shims.java?rev=801012&view=auto
==============================================================================
--- hadoop/hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Jetty20Shims.java (added)
+++ hadoop/hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Jetty20Shims.java Tue Aug 4 23:10:53 2009
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.shims;
+
+
+import org.mortbay.jetty.bio.SocketConnector;
+import org.mortbay.jetty.handler.RequestLogHandler;
+import org.mortbay.jetty.webapp.WebAppContext;
+
+import java.io.IOException;
+
+public class Jetty20Shims implements JettyShims {
+ public Server startServer(String listen, int port) throws IOException {
+ Server s = new Server();
+ s.setupListenerHostPort(listen, port);
+ return s;
+ }
+
+ private static class Server extends org.mortbay.jetty.Server implements JettyShims.Server {
+ public void addWar(String war, String contextPath) {
+ WebAppContext wac = new WebAppContext();
+ RequestLogHandler rlh = new RequestLogHandler();
+ rlh.setHandler(wac);
+ this.addHandler(rlh);
+ }
+
+ public void setupListenerHostPort(String listen, int port)
+ throws IOException {
+
+ SocketConnector connector = new SocketConnector();
+ connector.setPort(port);
+ connector.setHost(listen);
+ this.addConnector(connector);
+ }
+ }
+}
\ No newline at end of file
Added: hadoop/hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java?rev=801012&view=auto
==============================================================================
--- hadoop/hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java (added)
+++ hadoop/hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java Tue Aug 4 23:10:53 2009
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.shims;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import java.io.IOException;
+
+
+/**
+ * In order to be compatible with multiple versions of Hadoop, all parts
+ * of the Hadoop interface that are not cross-version compatible are
+ * encapsulated in an implementation of this class. Users should use
+ * the ShimLoader class as a factory to obtain an implementation of
+ * HadoopShims corresponding to the version of Hadoop currently on the
+ * classpath.
+ */
+public interface HadoopShims {
+
+ /**
+ * Return true if the current version of Hadoop uses the JobShell for
+ * command line interpretation.
+ */
+ public boolean usesJobShell();
+
+ /**
+ * Calls fs.deleteOnExit(path) if such a function exists.
+ *
+ * @returns true if the call was successful
+ */
+ public boolean fileSystemDeleteOnExit(FileSystem fs, Path path) throws IOException;
+
+ /**
+ * Calls fmt.validateInput(conf) if such a function exists.
+ */
+ public void inputFormatValidateInput(InputFormat fmt, JobConf conf) throws IOException;
+
+ /**
+ * If JobClient.getCommandLineConfig exists, sets the given
+ * property/value pair in that Configuration object.
+ *
+ * This applies for Hadoop 0.17 through 0.19
+ */
+ public void setTmpFiles(String prop, String files);
+
+ /**
+ * Returns a shim to wrap MiniDFSCluster. This is necessary since this class
+ * was moved from org.apache.hadoop.dfs to org.apache.hadoop.hdfs
+ */
+ public MiniDFSShim getMiniDfs(Configuration conf,
+ int numDataNodes,
+ boolean format,
+ String[] racks) throws IOException;
+
+ /**
+ * Shim around the functions in MiniDFSCluster that Hive uses.
+ */
+ public interface MiniDFSShim {
+ public FileSystem getFileSystem() throws IOException;
+ public void shutdown() throws IOException;
+ }
+}
Added: hadoop/hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/JettyShims.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/JettyShims.java?rev=801012&view=auto
==============================================================================
--- hadoop/hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/JettyShims.java (added)
+++ hadoop/hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/JettyShims.java Tue Aug 4 23:10:53 2009
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.shims;
+
+import java.io.IOException;
+
+/**
+ * Since Hadoop ships with different versions of Jetty in different versions,
+ * Hive uses a shim layer to access the parts of the API that have changed.
+ * Users should obtain an instance of this class using the ShimLoader factory.
+ */
+public interface JettyShims {
+
+ public Server startServer(String listen, int port) throws IOException;
+
+ public interface Server {
+ public void addWar(String war, String mount);
+ public void start() throws Exception;
+ public void join() throws java.lang.InterruptedException;
+ public void stop() throws Exception;
+ }
+}
\ No newline at end of file
Added: hadoop/hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/ShimLoader.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/ShimLoader.java?rev=801012&view=auto
==============================================================================
--- hadoop/hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/ShimLoader.java (added)
+++ hadoop/hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/ShimLoader.java Tue Aug 4 23:10:53 2009
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.shims;
+
+import org.apache.hadoop.util.VersionInfo;
+import java.util.Map;
+import java.util.HashMap;
+
+public abstract class ShimLoader {
+ private static HadoopShims hadoopShims;
+ private static JettyShims jettyShims;
+
+ /**
+ * The names of the classes for shimming Hadoop for each major version.
+ */
+ private static HashMap<String, String> HADOOP_SHIM_CLASSES =
+ new HashMap<String, String>();
+
+ static {
+ HADOOP_SHIM_CLASSES.put("0.17", "org.apache.hadoop.hive.shims.Hadoop17Shims");
+ HADOOP_SHIM_CLASSES.put("0.18", "org.apache.hadoop.hive.shims.Hadoop18Shims");
+ HADOOP_SHIM_CLASSES.put("0.19", "org.apache.hadoop.hive.shims.Hadoop19Shims");
+ HADOOP_SHIM_CLASSES.put("0.20", "org.apache.hadoop.hive.shims.Hadoop20Shims");
+ }
+
+ /**
+ * The names of the classes for shimming Jetty for each major version of
+ * Hadoop.
+ */
+ private static HashMap<String, String> JETTY_SHIM_CLASSES =
+ new HashMap<String, String>();
+
+ static {
+ JETTY_SHIM_CLASSES.put("0.17", "org.apache.jetty.hive.shims.Jetty17Shims");
+ JETTY_SHIM_CLASSES.put("0.18", "org.apache.jetty.hive.shims.Jetty18Shims");
+ JETTY_SHIM_CLASSES.put("0.19", "org.apache.jetty.hive.shims.Jetty19Shims");
+ JETTY_SHIM_CLASSES.put("0.20", "org.apache.jetty.hive.shims.Jetty20Shims");
+ }
+
+
+ /**
+ * Factory method to get an instance of HadoopShims based on the
+ * version of Hadoop on the classpath.
+ */
+ public synchronized static HadoopShims getHadoopShims() {
+ if (hadoopShims == null) {
+ hadoopShims = loadShims(HADOOP_SHIM_CLASSES, HadoopShims.class);
+ }
+ return hadoopShims;
+ }
+
+ /**
+ * Factory method to get an instance of JettyShims based on the version
+ * of Hadoop on the classpath.
+ */
+ public synchronized static JettyShims getJettyShims() {
+ if (jettyShims == null) {
+ jettyShims = loadShims(JETTY_SHIM_CLASSES, JettyShims.class);
+ }
+ return jettyShims;
+ }
+
+ @SuppressWarnings("unchecked")
+ private static <T> T loadShims(Map<String, String> classMap, Class<T> xface) {
+ String vers = getMajorVersion();
+ String className = classMap.get(vers);
+ try {
+ Class clazz = Class.forName(className);
+ return xface.cast(clazz.newInstance());
+ } catch (Exception e) {
+ throw new RuntimeException("Could not load shims in class " +
+ className, e);
+ }
+ }
+
+ /**
+ * Return the major version of Hadoop currently on the classpath.
+ * This is simply the first two components of the version number
+ * (e.g "0.18" or "0.20")
+ */
+ private static String getMajorVersion() {
+ String vers = VersionInfo.getVersion();
+
+ String parts[] = vers.split("\\.");
+ if (parts.length < 2) {
+ throw new RuntimeException("Illegal Hadoop Version: " + vers +
+ " (expected A.B.* format)");
+ }
+ return parts[0] + "." + parts[1];
+ }
+}
\ No newline at end of file