You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2012/07/01 23:59:44 UTC
svn commit: r1356035 - in /hive/trunk:
ant/src/org/apache/hadoop/hive/ant/DistinctElementsClassPath.java
build-common.xml jdbc/build.xml odbc/build.xml shims/build.xml
Author: hashutosh
Date: Sun Jul 1 21:59:43 2012
New Revision: 1356035
URL: http://svn.apache.org/viewvc?rev=1356035&view=rev
Log:
HIVE-3172 : Remove the duplicate JAR entries from the (“test.classpath”) to avoid command line exceeding char limit on windows (Kanna Karanam via Ashutosh Chauhan)
Added:
hive/trunk/ant/src/org/apache/hadoop/hive/ant/DistinctElementsClassPath.java
Modified:
hive/trunk/build-common.xml
hive/trunk/jdbc/build.xml
hive/trunk/odbc/build.xml
hive/trunk/shims/build.xml
Added: hive/trunk/ant/src/org/apache/hadoop/hive/ant/DistinctElementsClassPath.java
URL: http://svn.apache.org/viewvc/hive/trunk/ant/src/org/apache/hadoop/hive/ant/DistinctElementsClassPath.java?rev=1356035&view=auto
==============================================================================
--- hive/trunk/ant/src/org/apache/hadoop/hive/ant/DistinctElementsClassPath.java (added)
+++ hive/trunk/ant/src/org/apache/hadoop/hive/ant/DistinctElementsClassPath.java Sun Jul 1 21:59:43 2012
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hive.ant;
+
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.HashSet;
+
+import org.apache.tools.ant.Project;
+import org.apache.tools.ant.types.Path;
+
+/**
+ * This object represents a path as used by CLASSPATH or PATH environment variable. String
+ * representation of this object returns the path with unique elements to reduce the chances of
+ * exceeding the character limit problem on windows by removing if there are duplicate files(JARs)
+ * in the original class path.
+ */
+public class DistinctElementsClassPath extends Path {
+
+ /**
+ * Invoked by IntrospectionHelper for <code>setXXX(Path p)</code>
+ * attribute setters.
+ * @param p the <code>Project</code> for this path.
+ * @param path the <code>String</code> path definition.
+ */
+ public DistinctElementsClassPath(Project p, String path) {
+ super(p, path);
+ }
+
+ /**
+ * Construct an empty <code>Path</code>.
+ * @param project the <code>Project</code> for this path.
+ */
+ public DistinctElementsClassPath(Project project) {
+ super(project);
+ }
+
+ /**
+ * Returns the list of path elements after removing the duplicate files from the
+ * original Path
+ */
+ @Override
+ public String[] list() {
+ HashSet includedElements = new HashSet();
+ ArrayList resultElements = new ArrayList();
+ for(String pathElement : super.list()) {
+ if(pathElement != null && !pathElement.isEmpty()) {
+ File p = new File(pathElement);
+ if(p.exists()) {
+ String setItem = pathElement.toLowerCase();
+ if (p.isFile()) {
+ setItem = p.getName().toLowerCase();
+ }
+ if(!includedElements.contains(setItem)) {
+ includedElements.add(setItem);
+ resultElements.add(pathElement);
+ }
+ }
+ }
+ }
+
+ return (String[])resultElements.toArray (new String [resultElements.size ()]);
+ }
+
+ /**
+ * Returns a textual representation of the path after removing the duplicate files from the
+ * original Path.
+ */
+ @Override
+ public String toString() {
+ return org.apache.commons.lang.StringUtils.join(this.list(), File.pathSeparatorChar);
+ }
+}
\ No newline at end of file
Modified: hive/trunk/build-common.xml
URL: http://svn.apache.org/viewvc/hive/trunk/build-common.xml?rev=1356035&r1=1356034&r2=1356035&view=diff
==============================================================================
--- hive/trunk/build-common.xml (original)
+++ hive/trunk/build-common.xml Sun Jul 1 21:59:43 2012
@@ -69,31 +69,35 @@
<property name="hadoop.opts.23" value="-D mapreduce.framework.name=local" />
<property name="hadoop.opts.20" value="" />
- <path id="test.classpath">
- <pathelement location="${test.build.classes}" />
- <pathelement location="${test.build.resources}" />
- <pathelement location="" />
- <pathelement location="${test.src.data.dir}/conf"/>
- <pathelement location="${hive.conf.dir}"/>
- <pathelement location="${build.dir.hive}/cli/test/classes"/>
- <pathelement location="${build.dir.hive}/common/test/classes"/>
- <pathelement location="${build.dir.hive}/hbase-handler/test/classes"/>
- <pathelement location="${build.dir.hive}/hwi/test/classes"/>
- <pathelement location="${build.dir.hive}/jdbc/test/classes"/>
- <pathelement location="${build.dir.hive}/metastore/test/classes"/>
- <pathelement location="${build.dir.hive}/ql/test/classes"/>
- <pathelement location="${build.dir.hive}/serde/test/classes"/>
- <pathelement location="${build.dir.hive}/service/test/classes"/>
- <pathelement location="${build.dir.hive}/shims/test/classes"/>
-
- <!-- Include build/dist/lib on the classpath before Ivy and exclude hive jars from Ivy to make sure we get the local changes when we test Hive -->
- <fileset dir="${build.dir.hive}/dist/lib" includes="*.jar" erroronmissingdir="false" excludes="**/hive_contrib*.jar,**/hive-contrib*.jar"/>
- <fileset dir="${hive.root}/build/ivy/lib/test" includes="*.jar" erroronmissingdir="false" excludes="**/hive_*.jar,**/hive-*.jar"/>
- <fileset dir="${hive.root}/build/ivy/lib/default" includes="*.jar" erroronmissingdir="false" excludes="**/hive_*.jar,**/hive-*.jar" />
- <fileset dir="${hive.root}/testlibs" includes="*.jar"/>
- <fileset dir="${hive.root}/build/ivy/lib/hadoop0.${hadoop.mr.rev}.shim" includes="*.jar" />
- <pathelement location="${build.classes}" />
- </path>
+ <target name="set-test-classpath">
+ <typedef name="distinctelementsclasspath" classname="org.apache.hadoop.hive.ant.DistinctElementsClassPath"
+ classpath="${build.dir.hive}/anttasks/hive-anttasks-${version}.jar:${build.ivy.lib.dir}/default/commons-collections-${commons-collections.version}.jar:${build.ivy.lib.dir}/default/commons-lang-${commons-lang.version}.jar"/>
+ <distinctelementsclasspath id="test.classpath">
+ <pathelement location="${test.build.classes}" />
+ <pathelement location="${test.build.resources}" />
+ <pathelement location="" />
+ <pathelement location="${test.src.data.dir}/conf"/>
+ <pathelement location="${hive.conf.dir}"/>
+ <pathelement location="${build.dir.hive}/cli/test/classes"/>
+ <pathelement location="${build.dir.hive}/common/test/classes"/>
+ <pathelement location="${build.dir.hive}/hbase-handler/test/classes"/>
+ <pathelement location="${build.dir.hive}/hwi/test/classes"/>
+ <pathelement location="${build.dir.hive}/jdbc/test/classes"/>
+ <pathelement location="${build.dir.hive}/metastore/test/classes"/>
+ <pathelement location="${build.dir.hive}/ql/test/classes"/>
+ <pathelement location="${build.dir.hive}/serde/test/classes"/>
+ <pathelement location="${build.dir.hive}/service/test/classes"/>
+ <pathelement location="${build.dir.hive}/shims/test/classes"/>
+
+ <!-- Include build/dist/lib on the classpath before Ivy and exclude hive jars from Ivy to make sure we get the local changes when we test Hive -->
+ <fileset dir="${build.dir.hive}/dist/lib" includes="*.jar" erroronmissingdir="false" excludes="**/hive_contrib*.jar,**/hive-contrib*.jar,**/lib*.jar"/>
+ <fileset dir="${hive.root}/build/ivy/lib/test" includes="*.jar" erroronmissingdir="false" excludes="**/hive_*.jar,**/hive-*.jar"/>
+ <fileset dir="${hive.root}/build/ivy/lib/default" includes="*.jar" erroronmissingdir="false" excludes="**/hive_*.jar,**/hive-*.jar" />
+ <fileset dir="${hive.root}/testlibs" includes="*.jar"/>
+ <fileset dir="${hive.root}/build/ivy/lib/hadoop0.${hadoop.mr.rev}.shim" includes="*.jar" />
+ <pathelement location="${build.classes}" />
+ </distinctelementsclasspath>
+ </target>
<!-- include contrib on local classpath, but not on cluster -->
<!-- https://reviews.facebook.net/D2133#comment-47 -->
@@ -209,7 +213,7 @@
<echo message="Project: ${ant.project.name}"/>
</target>
- <target name="test-init">
+ <target name="test-init" depends="set-test-classpath">
<echo message="Project: ${ant.project.name}"/>
<mkdir dir="${test.data.dir}"/>
<mkdir dir="${test.log.dir}/clientpositive"/>
@@ -262,7 +266,7 @@
</target>
<!-- target to compile tests -->
- <target name="compile-test" depends="compile,ivy-retrieve-test">
+ <target name="compile-test" depends="compile,ivy-retrieve-test,set-test-classpath">
<echo message="Project: ${ant.project.name}"/>
<javac
encoding="${build.encoding}"
Modified: hive/trunk/jdbc/build.xml
URL: http://svn.apache.org/viewvc/hive/trunk/jdbc/build.xml?rev=1356035&r1=1356034&r2=1356035&view=diff
==============================================================================
--- hive/trunk/jdbc/build.xml (original)
+++ hive/trunk/jdbc/build.xml Sun Jul 1 21:59:43 2012
@@ -22,17 +22,19 @@
<import file="../build-common.xml"/>
- <path id="test.classpath">
- <pathelement location="${test.build.classes}" />
- <pathelement location="" />
- <pathelement location="${test.src.data.dir}/conf"/>
- <pathelement location="${hive.conf.dir}"/>
- <fileset dir="${test.src.data.dir}" includes="files/*.jar"/>
- <fileset dir="${hive.root}" includes="testlibs/*.jar"/>
- <pathelement location="${build.dir.hive}/ql/test/classes"/>
- <fileset dir="${hive.root}/build/ivy/lib/hadoop0.${hadoop.mr.rev}.shim" includes="*.jar" />
- <path refid="classpath" />
- </path>
+ <target name="set-test-classpath">
+ <path id="test.classpath">
+ <pathelement location="${test.build.classes}" />
+ <pathelement location="" />
+ <pathelement location="${test.src.data.dir}/conf"/>
+ <pathelement location="${hive.conf.dir}"/>
+ <fileset dir="${test.src.data.dir}" includes="files/*.jar"/>
+ <fileset dir="${hive.root}" includes="testlibs/*.jar"/>
+ <pathelement location="${build.dir.hive}/ql/test/classes"/>
+ <fileset dir="${hive.root}/build/ivy/lib/hadoop0.${hadoop.mr.rev}.shim" includes="*.jar" />
+ <path refid="classpath" />
+ </path>
+ </target>
<target name="compile" depends="init,ivy-retrieve">
<echo message="Project: ${ant.project.name}"/>
Modified: hive/trunk/odbc/build.xml
URL: http://svn.apache.org/viewvc/hive/trunk/odbc/build.xml?rev=1356035&r1=1356034&r2=1356035&view=diff
==============================================================================
--- hive/trunk/odbc/build.xml (original)
+++ hive/trunk/odbc/build.xml Sun Jul 1 21:59:43 2012
@@ -29,16 +29,18 @@
<property name="make.cmd" value="make"/>
<import file="../build-common.xml"/>
- <path id="test.classpath">
- <pathelement location="${test.build.classes}" />
- <pathelement location="" />
- <pathelement location="${test.src.data.dir}/conf"/>
- <pathelement location="${hive.conf.dir}"/>
- <fileset dir="${test.src.data.dir}" includes="files/*.jar"/>
- <fileset dir="${hive.root}" includes="testlibs/*.jar"/>
- <pathelement location="${build.dir.hive}/ql/test/classes"/>
- <path refid="classpath"/>
- </path>
+ <target name="set-test-classpath">
+ <path id="test.classpath">
+ <pathelement location="${test.build.classes}" />
+ <pathelement location="" />
+ <pathelement location="${test.src.data.dir}/conf"/>
+ <pathelement location="${hive.conf.dir}"/>
+ <fileset dir="${test.src.data.dir}" includes="files/*.jar"/>
+ <fileset dir="${hive.root}" includes="testlibs/*.jar"/>
+ <pathelement location="${build.dir.hive}/ql/test/classes"/>
+ <path refid="classpath"/>
+ </path>
+ </target>
<target name="check-word-size">
<echo message="Project: ${ant.project.name}"/>
@@ -92,7 +94,7 @@
</target>
<!-- Only run tests if thrift.home is defined so that we don't break other tests -->
- <target name="test" depends="check-word-size,check-thrift-home" if="thrift.home.defined">
+ <target name="test" depends="check-word-size,check-thrift-home,set-test-classpath" if="thrift.home.defined">
<echo message="Project: ${ant.project.name}"/>
<exec dir="." executable="${make.cmd}" failonerror="true">
<arg line="test"/>
Modified: hive/trunk/shims/build.xml
URL: http://svn.apache.org/viewvc/hive/trunk/shims/build.xml?rev=1356035&r1=1356034&r2=1356035&view=diff
==============================================================================
--- hive/trunk/shims/build.xml (original)
+++ hive/trunk/shims/build.xml Sun Jul 1 21:59:43 2012
@@ -40,17 +40,18 @@ to call at top-level: ant deploy-contrib
<property name="shims.0.20S.hadoop.ivy.dir" value="${hive.root}/build/ivy/lib/hadoop0.20S.shim" />
<property name="shims.0.23.hadoop.ivy.dir" value="${hive.root}/build/ivy/lib/hadoop0.23.shim" />
- <path id="test.classpath">
- <pathelement location="${test.build.classes}" />
- <pathelement location="" />
- <pathelement location="${test.src.data.dir}/conf"/>
- <pathelement location="${hive.conf.dir}"/>
- <!-- We are running unit tests like the one inside ql -->
- <fileset dir="${hive.root}/testlibs" includes="*.jar" />
- <fileset dir="${shims.0.20S.hadoop.ivy.dir}" includes="*.jar" excludes="slf4j-*.jar" />
- <fileset dir="../lib" includes="*.jar" />
- <path refid="classpath" />
- </path>
+ <target name="set-test-classpath">
+ <path id="test.classpath">
+ <pathelement location="${test.build.classes}" />
+ <pathelement location="" />
+ <pathelement location="${test.src.data.dir}/conf"/>
+ <pathelement location="${hive.conf.dir}"/>
+ <fileset dir="${hive.root}/testlibs" includes="*.jar" />
+ <fileset dir="${shims.0.20S.hadoop.ivy.dir}" includes="*.jar" excludes="slf4j-*.jar" />
+ <fileset dir="../lib" includes="*.jar" />
+ <path refid="classpath" />
+ </path>
+ </target>
<target name="build_shims"
description="Build shims against a particular hadoop version">
@@ -95,7 +96,8 @@ to call at top-level: ant deploy-contrib
</for>
</target>
- <target name="compile_secure_test"
+ <target name="compile_secure_test"
+ depends="set-test-classpath"
description="Test shims against a particular hadoop version">
<echo message="Project: ${ant.project.name}"/>
<getversionpref property="hadoop.version.ant-internal.prefix" input="${hadoop.version.ant-internal}" />