You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by na...@apache.org on 2009/12/28 21:04:43 UTC
svn commit: r894240 - in /hadoop/hive/trunk: CHANGES.txt
ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java build-common.xml
contrib/build.xml ql/build.xml
Author: namit
Date: Mon Dec 28 20:04:42 2009
New Revision: 894240
URL: http://svn.apache.org/viewvc?rev=894240&view=rev
Log:
HIVE-1012. Ability to run testcases via regular expression
(Carl Steinbach via namit)
Modified:
hadoop/hive/trunk/CHANGES.txt
hadoop/hive/trunk/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java
hadoop/hive/trunk/build-common.xml
hadoop/hive/trunk/contrib/build.xml
hadoop/hive/trunk/ql/build.xml
Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=894240&r1=894239&r2=894240&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Mon Dec 28 20:04:42 2009
@@ -146,6 +146,9 @@
HIVE-980. update readme.txt to build hive using eclipse
(Patrick Angeles via namit)
+ HIVE-1012. Ability to run testcases via regular expression
+ (Carl Steinbach via namit)
+
OPTIMIZATIONS
BUG FIXES
Modified: hadoop/hive/trunk/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java?rev=894240&r1=894239&r2=894240&view=diff
==============================================================================
--- hadoop/hive/trunk/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java (original)
+++ hadoop/hive/trunk/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java Mon Dec 28 20:04:42 2009
@@ -18,10 +18,19 @@
package org.apache.hadoop.hive.ant;
-import java.io.*;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.io.FileWriter;
+import java.io.IOException;
import java.util.StringTokenizer;
import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.regex.Pattern;
+import org.apache.commons.lang.StringUtils;
import org.apache.tools.ant.AntClassLoader;
import org.apache.tools.ant.BuildException;
import org.apache.tools.ant.Task;
@@ -48,26 +57,44 @@
}
}
+
+ public class QFileRegexFilter extends QFileFilter {
+ Pattern filterPattern;
+
+ public QFileRegexFilter(String filter) {
+ filterPattern = Pattern.compile(filter);
+ }
+
+ public boolean accept(File filePath) {
+ if (!super.accept(filePath)) {
+ return false;
+ }
+ String testName = StringUtils.chomp(filePath.getName(), ".q");
+ return filterPattern.matcher(testName).matches();
+ }
+ }
- protected String templatePath;
-
- protected String outputDirectory;
+ private List<String> templatePaths = new ArrayList<String>();
+
+ private String outputDirectory;
- protected String queryDirectory;
+ private String queryDirectory;
- protected String queryFile;
+ private String queryFile;
+
+ private String queryFileRegex;
- protected String resultsDirectory;
+ private String resultsDirectory;
- protected String logDirectory;
+ private String logDirectory;
- protected String template;
+ private String template;
- protected String className;
+ private String className;
- protected String logFile;
+ private String logFile;
- protected String clusterMode;
+ private String clusterMode;
public void setClusterMode(String clusterMode) {
this.clusterMode = clusterMode;
@@ -102,30 +129,21 @@
}
public void setTemplatePath(String templatePath) throws Exception {
- StringBuffer resolvedPath = new StringBuffer();
- StringTokenizer st = new StringTokenizer(templatePath, ",");
- while (st.hasMoreTokens()) {
- // resolve relative path from basedir and leave
- // absolute path untouched.
- File fullPath = project.resolveFile(st.nextToken());
- resolvedPath.append(fullPath.getCanonicalPath());
- if (st.hasMoreTokens()) {
- resolvedPath.append(",");
- }
+ templatePaths.clear();
+ for (String relativePath : templatePath.split(",")) {
+ templatePaths.add(project.resolveFile(relativePath).getCanonicalPath());
}
- this.templatePath = resolvedPath.toString();
- System.out.println("Template Path:" + this.templatePath);
+ System.out.println("Template Path:" + getTemplatePath());
}
public String getTemplatePath() {
- return templatePath;
+ return StringUtils.join(templatePaths, ",");
}
public void setOutputDirectory(File outputDirectory) {
try {
this.outputDirectory = outputDirectory.getCanonicalPath();
- }
- catch (IOException ioe) {
+ } catch (IOException ioe) {
throw new BuildException(ioe);
}
}
@@ -139,7 +157,7 @@
}
public String getLogDirectory() {
- return this.logDirectory;
+ return logDirectory;
}
public void setResultsDirectory(String resultsDirectory) {
@@ -147,7 +165,7 @@
}
public String getResultsDirectory() {
- return this.resultsDirectory;
+ return resultsDirectory;
}
public void setQueryDirectory(String queryDirectory) {
@@ -155,7 +173,7 @@
}
public String getQueryDirectory() {
- return this.queryDirectory;
+ return queryDirectory;
}
public void setQueryFile(String queryFile) {
@@ -163,12 +181,20 @@
}
public String getQueryFile() {
- return this.queryFile;
+ return queryFile;
+ }
+
+ public void setQueryFileRegex(String queryFileRegex) {
+ this.queryFileRegex = queryFileRegex;
+ }
+
+ public String getQueryFileRegex() {
+ return queryFileRegex;
}
public void execute() throws BuildException {
- if (templatePath == null) {
+ if (getTemplatePath().equals("")) {
throw new BuildException("No templatePath attribute specified");
}
@@ -196,7 +222,7 @@
throw new BuildException("No className specified");
}
- File [] qFiles = null;
+ List<File> qFiles = new ArrayList<File>();
File outDir = null;
File resultsDir = null;
File logDir = null;
@@ -209,17 +235,20 @@
if (queryFile != null && !queryFile.equals("")) {
// The user may have passed a list of files - comma seperated
- String[] queryFiles = queryFile.split(",");
- qFiles = new File[queryFiles.length];
-
- for (int i = 0; i < queryFiles.length; i++) {
- qFiles[i] = inpDir != null ? new File(inpDir, queryFiles[i]) : new File(queryFiles[i]);
+ for (String qFile : queryFile.split(",")) {
+ if (null != inpDir) {
+ qFiles.add(new File(inpDir, qFile));
+ } else {
+ qFiles.add(new File(qFile));
+ }
}
+ } else if (queryFileRegex != null && !queryFileRegex.equals("")) {
+ qFiles.addAll(Arrays.asList(inpDir.listFiles(new QFileRegexFilter(queryFileRegex))));
+ } else {
+ qFiles.addAll(Arrays.asList(inpDir.listFiles(new QFileFilter())));
}
- else {
- qFiles = inpDir.listFiles(new QFileFilter());
- Arrays.sort(qFiles);
- }
+
+ Collections.sort(qFiles);
// Make sure the output directory exists, if it doesn't
// then create it.
@@ -237,15 +266,14 @@
if (!resultsDir.exists()) {
throw new BuildException("Results Directory " + resultsDir.getCanonicalPath() + " does not exist");
}
- }
- catch (Exception e) {
+ } catch (Exception e) {
throw new BuildException(e);
}
VelocityEngine ve = new VelocityEngine();
try {
- ve.setProperty(RuntimeConstants.FILE_RESOURCE_LOADER_PATH, templatePath);
+ ve.setProperty(RuntimeConstants.FILE_RESOURCE_LOADER_PATH, getTemplatePath());
if (logFile != null) {
File lf = new File(logFile);
if (lf.exists()) {
@@ -260,8 +288,9 @@
ve.init();
Template t = ve.getTemplate(template);
- if (clusterMode == null)
+ if (clusterMode == null) {
clusterMode = new String("");
+ }
// For each of the qFiles generate the test
VelocityContext ctx = new VelocityContext();
@@ -277,22 +306,17 @@
writer.close();
System.out.println("Generated " + outFile.getCanonicalPath() + " from template " + template);
- }
- catch(BuildException e) {
+ } catch(BuildException e) {
throw e;
- }
- catch(MethodInvocationException e) {
+ } catch(MethodInvocationException e) {
throw new BuildException("Exception thrown by '" + e.getReferenceName() + "." +
e.getMethodName() +"'",
e.getWrappedThrowable());
- }
- catch(ParseErrorException e) {
+ } catch(ParseErrorException e) {
throw new BuildException("Velocity syntax error", e);
- }
- catch(ResourceNotFoundException e) {
+ } catch(ResourceNotFoundException e) {
throw new BuildException("Resource not found", e);
- }
- catch(Exception e) {
+ } catch(Exception e) {
throw new BuildException("Generation failed", e);
}
}
Modified: hadoop/hive/trunk/build-common.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/build-common.xml?rev=894240&r1=894239&r2=894240&view=diff
==============================================================================
--- hadoop/hive/trunk/build-common.xml (original)
+++ hadoop/hive/trunk/build-common.xml Mon Dec 28 20:04:42 2009
@@ -236,6 +236,12 @@
</not>
</condition>
+ <condition property="qfile_regex" value="">
+ <not>
+ <isset property="qfile_regex"/>
+ </not>
+ </condition>
+
<condition property="overwrite" value="false">
<not>
<isset property="overwrite"/>
Modified: hadoop/hive/trunk/contrib/build.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/contrib/build.xml?rev=894240&r1=894239&r2=894240&view=diff
==============================================================================
--- hadoop/hive/trunk/contrib/build.xml (original)
+++ hadoop/hive/trunk/contrib/build.xml Mon Dec 28 20:04:42 2009
@@ -71,6 +71,7 @@
templatePath="${ql.test.template.dir}" template="TestParse.vm"
queryDirectory="${contrib.test.query.dir}/positive"
queryFile="${qfile}"
+ queryFileRegex="${qfile_regex}"
resultsDirectory="${contrib.test.results.dir}/compiler" className="TestContribParse"
logFile="${test.log.dir}/testcontribparsegen.log"
logDirectory="${test.log.dir}/contribpositive"/>
@@ -79,6 +80,7 @@
templatePath="${ql.test.template.dir}" template="TestParseNegative.vm"
queryDirectory="${contrib.test.query.dir}/negative"
queryFile="${qfile}"
+ queryFileRegex="${qfile_regex}"
resultsDirectory="${contrib.test.results.dir}/compiler/errors" className="TestContribParseNegative"
logFile="${test.log.dir}/testcontribparseneggen.log"
logDirectory="${test.log.dir}/contribnegative"/>
@@ -87,6 +89,7 @@
templatePath="${ql.test.template.dir}" template="TestCliDriver.vm"
queryDirectory="${contrib.test.query.dir}/clientpositive"
queryFile="${qfile}"
+ queryFileRegex="${qfile_regex}"
clusterMode="${clustermode}"
resultsDirectory="${contrib.test.results.dir}/clientpositive" className="TestContribCliDriver"
logFile="${test.log.dir}/testcontribclidrivergen.log"
@@ -96,6 +99,7 @@
templatePath="${ql.test.template.dir}" template="TestNegativeCliDriver.vm"
queryDirectory="${contrib.test.query.dir}/clientnegative"
queryFile="${qfile}"
+ queryFileRegex="${qfile_regex}"
resultsDirectory="${contrib.test.results.dir}/clientnegative" className="TestContribNegativeCliDriver"
logFile="${test.log.dir}/testcontribnegclidrivergen.log"
logDirectory="${test.log.dir}/contribclientnegative"/>
Modified: hadoop/hive/trunk/ql/build.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/build.xml?rev=894240&r1=894239&r2=894240&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/build.xml (original)
+++ hadoop/hive/trunk/ql/build.xml Mon Dec 28 20:04:42 2009
@@ -66,6 +66,7 @@
templatePath="${ql.test.template.dir}" template="TestParse.vm"
queryDirectory="${ql.test.query.dir}/positive"
queryFile="${qfile}"
+ queryFileRegex="${qfile_regex}"
resultsDirectory="${ql.test.results.dir}/compiler" className="TestParse"
logFile="${test.log.dir}/testparsegen.log"
logDirectory="${test.log.dir}/positive"/>
@@ -74,6 +75,7 @@
templatePath="${ql.test.template.dir}" template="TestParseNegative.vm"
queryDirectory="${ql.test.query.dir}/negative"
queryFile="${qfile}"
+ queryFileRegex="${qfile_regex}"
resultsDirectory="${ql.test.results.dir}/compiler/errors" className="TestParseNegative"
logFile="${test.log.dir}/testparseneggen.log"
logDirectory="${test.log.dir}/negative"/>
@@ -82,6 +84,7 @@
templatePath="${ql.test.template.dir}" template="TestCliDriver.vm"
queryDirectory="${ql.test.query.clientpositive.dir}"
queryFile="${qfile}"
+ queryFileRegex="${qfile_regex}"
clusterMode="${clustermode}"
resultsDirectory="${ql.test.results.clientpositive.dir}" className="TestCliDriver"
logFile="${test.log.dir}/testclidrivergen.log"
@@ -91,6 +94,7 @@
templatePath="${ql.test.template.dir}" template="TestNegativeCliDriver.vm"
queryDirectory="${ql.test.query.dir}/clientnegative"
queryFile="${qfile}"
+ queryFileRegex="${qfile_regex}"
resultsDirectory="${ql.test.results.dir}/clientnegative" className="TestNegativeCliDriver"
logFile="${test.log.dir}/testnegclidrivergen.log"
logDirectory="${test.log.dir}/clientnegative"/>