You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by na...@apache.org on 2010/08/25 23:07:55 UTC
svn commit: r989372 - in /hadoop/hive/trunk: CHANGES.txt
ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java build-common.xml
build.xml data/conf/hive-site.xml ql/build.xml
ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java
Author: namit
Date: Wed Aug 25 21:07:55 2010
New Revision: 989372
URL: http://svn.apache.org/viewvc?rev=989372&view=rev
Log:
HIVE-1523. Enable some tests in miniMR mode
(Joydeep Sen Sarma via namit)
M ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java
M build.xml
M data/conf/hive-site.xml
M build-common.xml
M CHANGES.txt
M ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java
M ql/build.xml
Modified:
hadoop/hive/trunk/CHANGES.txt
hadoop/hive/trunk/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java
hadoop/hive/trunk/build-common.xml
hadoop/hive/trunk/build.xml
hadoop/hive/trunk/data/conf/hive-site.xml
hadoop/hive/trunk/ql/build.xml
hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java
Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=989372&r1=989371&r2=989372&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Wed Aug 25 21:07:55 2010
@@ -105,6 +105,9 @@ Trunk - Unreleased
HIVE-1441. Extend ivy offline mode to cover metastore downloads
(John Sichi via Ning Zhang)
+ HIVE-1523. Enable some tests in miniMR mode
+ (Joydeep Sen Sarma via namit)
+
OPTIMIZATIONS
BUG FIXES
Modified: hadoop/hive/trunk/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java?rev=989372&r1=989371&r2=989372&view=diff
==============================================================================
--- hadoop/hive/trunk/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java (original)
+++ hadoop/hive/trunk/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java Wed Aug 25 21:07:55 2010
@@ -81,6 +81,8 @@ public class QTestGenTask extends Task {
private String queryDirectory;
private String queryFile;
+
+ private String excludeQueryFile;
private String queryFileRegex;
@@ -193,11 +195,19 @@ public class QTestGenTask extends Task {
public String getQueryFile() {
return queryFile;
}
-
+
+ public void setExcludeQueryFile(String excludeQueryFile) {
+ this.excludeQueryFile = excludeQueryFile;
+ }
+
+ public String getExcludeQueryFile() {
+ return excludeQueryFile;
+ }
+
public void setQueryFileRegex(String queryFileRegex) {
this.queryFileRegex = queryFileRegex;
}
-
+
public String getQueryFileRegex() {
return queryFileRegex;
}
@@ -257,6 +267,17 @@ public class QTestGenTask extends Task {
} else {
qFiles.addAll(Arrays.asList(inpDir.listFiles(new QFileFilter())));
}
+
+ if (excludeQueryFile != null && !excludeQueryFile.equals("")) {
+ // Exclude specified query files, comma separated
+ for (String qFile : excludeQueryFile.split(",")) {
+ if (null != inpDir) {
+ qFiles.remove(new File(inpDir, qFile));
+ } else {
+ qFiles.remove(new File(qFile));
+ }
+ }
+ }
Collections.sort(qFiles);
Modified: hadoop/hive/trunk/build-common.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/build-common.xml?rev=989372&r1=989371&r2=989372&view=diff
==============================================================================
--- hadoop/hive/trunk/build-common.xml (original)
+++ hadoop/hive/trunk/build-common.xml Wed Aug 25 21:07:55 2010
@@ -50,7 +50,7 @@
<property name="test.timeout" value="13200000"/>
<property name="test.junit.output.format" value="xml"/>
<property name="test.junit.output.usefile" value="true"/>
- <property name="minimr.query.files" value="join1.q,groupby1.q"/>
+ <property name="minimr.query.files" value="input16_cc.q,scriptfile1.q,bucket4.q,bucketmapjoin6.q,disable_merge_for_bucketing.q,reduce_deduplicate.q,smb_mapjoin_8.q,join1.q,groupby2.q,bucketizedhiveinputformat.q"/>
<property name="test.silent" value="true"/>
<property name="hadoopVersion" value="${hadoop.version.ant-internal}"/>
<property name="test.serialize.qplan" value="false"/>
@@ -330,16 +330,6 @@
<target name="test-conditions">
- <condition property="qfile" value="${minimr.query.files}">
- <and>
- <not>
- <isset property="qfile"/>
- </not>
-
- <equals arg1="${clustermode}" arg2="miniMR" />
- </and>
- </condition>
-
<condition property="qfile" value="">
<not>
<isset property="qfile"/>
@@ -399,6 +389,9 @@
</not>
</condition>
+ <property name="test.warehouse.dir" value="${test.warehouse.scheme}${build.dir}/test/data/warehouse"/>
+ <property name="mapred.job.tracker" value="local"/>
+ <property name="fs.default.name" value="file:///"/>
<!-- target to run the tests -->
<target name="test"
@@ -425,7 +418,9 @@
<sysproperty key="hadoop.log.dir" value="${test.log.dir}"/>
<sysproperty key="test.silent" value="${test.silent}"/>
<sysproperty key="test.tmp.dir" value="${build.dir}/tmp"/>
- <sysproperty key="test.warehouse.dir" value="${test.warehouse.scheme}${build.dir}/test/data/warehouse"/>
+ <sysproperty key="test.warehouse.dir" value="${test.warehouse.dir}"/>
+ <sysproperty key="mapred.job.tracker" value="${mapred.job.tracker}"/>
+ <sysproperty key="fs.default.name" value="${fs.default.name}"/>
<sysproperty key="build.dir" value="${build.dir}"/>
<sysproperty key="build.dir.hive" value="${build.dir.hive}"/>
Modified: hadoop/hive/trunk/build.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/build.xml?rev=989372&r1=989371&r2=989372&view=diff
==============================================================================
--- hadoop/hive/trunk/build.xml (original)
+++ hadoop/hive/trunk/build.xml Wed Aug 25 21:07:55 2010
@@ -342,16 +342,16 @@
<getversionpref property="hadoop.version.ant-internal.prefix" input="${hadoop.version.ant-internal}"/>
<condition property="jetty.jar" value="jetty-6.1.14.jar" else="jetty-5.1.4.jar">
- <equals arg1="${hadoop.version.ant-internal.prefix}" arg2="0.20" />
+ <matches string="${hadoop.version.ant-internal}" pattern="^0\.20\..*" />
</condition>
<condition property="jetty.util.jar" value="jetty-util-6.1.14.jar" else="jetty-ext/jsp-api.jar">
<!-- hadoop 0.19 and earlier does not have jetty-util.jar, so we just load another jar -->
- <equals arg1="${hadoop.version.ant-internal.prefix}" arg2="0.20" />
+ <matches string="${hadoop.version.ant-internal}" pattern="^0\.20\..*" />
</condition>
<condition property="servlet-api.jar" value="servlet-api-2.5-6.1.14.jar" else="servlet-api.jar">
- <equals arg1="${hadoop.version.ant-internal.prefix}" arg2="0.20" />
+ <matches string="${hadoop.version.ant-internal}" pattern="^0\.20\..*" />
</condition>
<pathconvert property="eclipse.project">
Modified: hadoop/hive/trunk/data/conf/hive-site.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/data/conf/hive-site.xml?rev=989372&r1=989371&r2=989372&view=diff
==============================================================================
--- hadoop/hive/trunk/data/conf/hive-site.xml (original)
+++ hadoop/hive/trunk/data/conf/hive-site.xml Wed Aug 25 21:07:55 2010
@@ -73,18 +73,6 @@
</property>
<property>
- <name>fs.default.name</name>
- <value>file:///</value>
- <description></description>
-</property>
-
-<property>
- <name>mapred.job.tracker</name>
- <value>local</value>
- <description></description>
-</property>
-
-<property>
<name>test.log.dir</name>
<value>${build.dir}/test/logs</value>
<description></description>
Modified: hadoop/hive/trunk/ql/build.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/build.xml?rev=989372&r1=989371&r2=989372&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/build.xml (original)
+++ hadoop/hive/trunk/ql/build.xml Wed Aug 25 21:07:55 2010
@@ -31,6 +31,7 @@
<property name="thrift.args" value="--gen java:beans --gen php --gen py -I ${basedir}/include -I ${basedir}/.. -o ${basedir}/src"/>
<import file="../build-common.xml"/>
+ <property name="ql.test.clientpositive.exclude" value="${minimr.query.files}"/>
<path id="test.classpath">
<pathelement location="${test.build.classes}" />
@@ -48,6 +49,7 @@
<pathelement location="${common.jar}"/>
<fileset dir="${hive.root}" includes="testlibs/*.jar"/>
<fileset dir="${hadoop.root}/lib" includes="*.jar"/>
+ <fileset dir="${hadoop.root}/lib/jsp-2.1" includes="*.jar"/>
<path refid="classpath"/>
</path>
@@ -86,6 +88,7 @@
templatePath="${ql.test.template.dir}" template="TestCliDriver.vm"
queryDirectory="${ql.test.query.clientpositive.dir}"
queryFile="${qfile}"
+ excludeQueryFile="${ql.test.clientpositive.exclude}"
queryFileRegex="${qfile_regex}"
clusterMode="${clustermode}"
resultsDirectory="${ql.test.results.clientpositive.dir}" className="TestCliDriver"
@@ -95,6 +98,18 @@
/>
<qtestgen outputDirectory="${test.build.src}/org/apache/hadoop/hive/cli"
+ templatePath="${ql.test.template.dir}" template="TestCliDriver.vm"
+ queryDirectory="${ql.test.query.clientpositive.dir}"
+ queryFile="${minimr.query.files}"
+ queryFileRegex="${qfile_regex}"
+ clusterMode="miniMR"
+ resultsDirectory="${ql.test.results.clientpositive.dir}" className="TestMinimrCliDriver"
+ logFile="${test.log.dir}/testminimrclidrivergen.log"
+ logDirectory="${test.log.dir}/clientpositive"
+ hadoopVersion="${hadoopVersion}"
+ />
+
+ <qtestgen outputDirectory="${test.build.src}/org/apache/hadoop/hive/cli"
templatePath="${ql.test.template.dir}" template="TestNegativeCliDriver.vm"
queryDirectory="${ql.test.query.dir}/clientnegative"
queryFile="${qfile}"
Modified: hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java?rev=989372&r1=989371&r2=989372&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java (original)
+++ hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java Wed Aug 25 21:07:55 2010
@@ -194,14 +194,17 @@ public class QTestUtil {
return null;
}
- public void initConf() {
+ public void initConf() throws Exception {
if (miniMr) {
- String fsName = conf.get("fs.default.name");
- assert fsName != null;
- // hive.metastore.warehouse.dir needs to be set relative to the jobtracker
- conf.set("hive.metastore.warehouse.dir", fsName
- .concat("/build/ql/test/data/warehouse/"));
- conf.set("mapred.job.tracker", "localhost:" + mr.getJobTrackerPort());
+ assert dfs != null;
+ assert mr != null;
+ // set fs.default.name to the uri of mini-dfs
+ conf.setVar(HiveConf.ConfVars.HADOOPFS, dfs.getFileSystem().getUri().toString());
+ // hive.metastore.warehouse.dir needs to be set relative to the mini-dfs
+ conf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE,
+ (new Path(dfs.getFileSystem().getUri().toString(),
+ "/build/ql/test/data/warehouse/")).toString());
+ conf.setVar(HiveConf.ConfVars.HADOOPJT, "localhost:" + mr.getJobTrackerPort());
}
}
@@ -330,7 +333,6 @@ public class QTestUtil {
}
public void cleanUp() throws Exception {
- String warehousePath = ((new URI(testWarehouse)).getPath());
// Drop any tables that remain due to unsuccessful runs
for (String s : new String[] {"src", "src1", "src_json", "src_thrift",
"src_sequencefile", "srcpart", "srcbucket", "srcbucket2", "dest1",
@@ -338,9 +340,15 @@ public class QTestUtil {
"dest_g1", "dest_g2", "fetchtask_ioexception"}) {
db.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, s);
}
- for (String s : new String[] {"dest4.out", "union.out"}) {
- deleteDirectory(new File(warehousePath, s));
+
+ // delete any contents in the warehouse dir
+ Path p = new Path(testWarehouse);
+ FileSystem fs = p.getFileSystem(conf);
+ FileStatus [] ls = fs.listStatus(p);
+ for (int i=0; (ls != null) && (i<ls.length); i++) {
+ fs.delete(ls[i].getPath(), true);
}
+
FunctionRegistry.unregisterTemporaryUDF("test_udaf");
FunctionRegistry.unregisterTemporaryUDF("test_error");
setup.tearDown();
@@ -516,7 +524,7 @@ public class QTestUtil {
}
public void cliInit(String tname, boolean recreate) throws Exception {
- if (miniMr || recreate) {
+ if (recreate) {
cleanUp();
createSources();
}
@@ -538,6 +546,7 @@ public class QTestUtil {
oldSs.out.close();
}
SessionState.start(ss);
+
cliDriver = new CliDriver();
if (tname.equals("init_file.q")) {
ss.initFiles.add("../data/scripts/test_init_file.sql");
@@ -859,6 +868,7 @@ public class QTestUtil {
"diff", "-a",
"-I", "file:",
"-I", "pfile:",
+ "-I", "hdfs:",
"-I", "/tmp/",
"-I", "invalidscheme:",
"-I", "lastUpdateTime",