You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sz...@apache.org on 2014/08/09 03:34:24 UTC
svn commit: r1616913 - in /hive/trunk: ant/src/org/apache/hadoop/hive/ant/
data/conf/ data/conf/tez/ data/scripts/ hbase-handler/src/test/templates/
itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/ itests/qtest/
itests/util/src/main/java/org/a...
Author: szehon
Date: Sat Aug 9 01:34:24 2014
New Revision: 1616913
URL: http://svn.apache.org/r1616913
Log:
HIVE-7519 : Refactor QTestUtil to remove its duplication with QFileClient for qtest setup and teardown (Ashish via Szehon)
Modified:
hive/trunk/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java
hive/trunk/data/conf/hive-site.xml
hive/trunk/data/conf/tez/hive-site.xml
hive/trunk/data/scripts/q_test_cleanup.sql
hive/trunk/data/scripts/q_test_init.sql
hive/trunk/hbase-handler/src/test/templates/TestHBaseCliDriver.vm
hive/trunk/hbase-handler/src/test/templates/TestHBaseNegativeCliDriver.vm
hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java
hive/trunk/itests/qtest/pom.xml
hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java
hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
hive/trunk/ql/src/test/templates/TestCliDriver.vm
hive/trunk/ql/src/test/templates/TestCompareCliDriver.vm
hive/trunk/ql/src/test/templates/TestNegativeCliDriver.vm
hive/trunk/ql/src/test/templates/TestParse.vm
hive/trunk/ql/src/test/templates/TestParseNegative.vm
Modified: hive/trunk/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java?rev=1616913&r1=1616912&r2=1616913&view=diff
==============================================================================
--- hive/trunk/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java (original)
+++ hive/trunk/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java Sat Aug 9 01:34:24 2014
@@ -141,6 +141,10 @@ public class QTestGenTask extends Task {
private String hadoopVersion;
+ private String initScript;
+
+ private String cleanupScript;
+
public void setHadoopVersion(String ver) {
this.hadoopVersion = ver;
}
@@ -197,6 +201,22 @@ public class QTestGenTask extends Task {
return template;
}
+ public String getInitScript() {
+ return initScript;
+ }
+
+ public void setInitScript(String initScript) {
+ this.initScript = initScript;
+ }
+
+ public String getCleanupScript() {
+ return cleanupScript;
+ }
+
+ public void setCleanupScript(String cleanupScript) {
+ this.cleanupScript = cleanupScript;
+ }
+
public void setHiveRootDirectory(File hiveRootDirectory) {
try {
this.hiveRootDirectory = hiveRootDirectory.getCanonicalPath();
@@ -444,6 +464,8 @@ public class QTestGenTask extends Task {
ctx.put("clusterMode", clusterMode);
ctx.put("hiveConfDir", escapePath(hiveConfDir));
ctx.put("hadoopVersion", hadoopVersion);
+ ctx.put("initScript", initScript);
+ ctx.put("cleanupScript", cleanupScript);
File outFile = new File(outDir, className + ".java");
FileWriter writer = new FileWriter(outFile);
Modified: hive/trunk/data/conf/hive-site.xml
URL: http://svn.apache.org/viewvc/hive/trunk/data/conf/hive-site.xml?rev=1616913&r1=1616912&r2=1616913&view=diff
==============================================================================
--- hive/trunk/data/conf/hive-site.xml (original)
+++ hive/trunk/data/conf/hive-site.xml Sat Aug 9 01:34:24 2014
@@ -112,6 +112,12 @@
</property>
<property>
+ <name>test.data.scripts</name>
+ <value>${hive.root}/data/scripts</value>
+ <description></description>
+</property>
+
+<property>
<name>hive.jar.path</name>
<value>${maven.local.repository}/org/apache/hive/hive-exec/${hive.version}/hive-exec-${hive.version}.jar</value>
<description></description>
Modified: hive/trunk/data/conf/tez/hive-site.xml
URL: http://svn.apache.org/viewvc/hive/trunk/data/conf/tez/hive-site.xml?rev=1616913&r1=1616912&r2=1616913&view=diff
==============================================================================
Binary files - no diff available.
Modified: hive/trunk/data/scripts/q_test_cleanup.sql
URL: http://svn.apache.org/viewvc/hive/trunk/data/scripts/q_test_cleanup.sql?rev=1616913&r1=1616912&r2=1616913&view=diff
==============================================================================
--- hive/trunk/data/scripts/q_test_cleanup.sql (original)
+++ hive/trunk/data/scripts/q_test_cleanup.sql Sat Aug 9 01:34:24 2014
@@ -7,4 +7,12 @@ DROP TABLE IF EXISTS srcbucket;
DROP TABLE IF EXISTS srcbucket2;
DROP TABLE IF EXISTS srcpart;
DROP TABLE IF EXISTS primitives;
-
+DROP TABLE IF EXISTS dest1;
+DROP TABLE IF EXISTS dest2;
+DROP TABLE IF EXISTS dest3;
+DROP TABLE IF EXISTS dest4;
+DROP TABLE IF EXISTS dest4_sequencefile;
+DROP TABLE IF EXISTS dest_j1;
+DROP TABLE IF EXISTS dest_g1;
+DROP TABLE IF EXISTS dest_g2;
+DROP TABLE IF EXISTS fetchtask_ioexception;
Modified: hive/trunk/data/scripts/q_test_init.sql
URL: http://svn.apache.org/viewvc/hive/trunk/data/scripts/q_test_init.sql?rev=1616913&r1=1616912&r2=1616913&view=diff
==============================================================================
--- hive/trunk/data/scripts/q_test_init.sql (original)
+++ hive/trunk/data/scripts/q_test_init.sql Sat Aug 9 01:34:24 2014
@@ -3,7 +3,7 @@
--
DROP TABLE IF EXISTS src;
-CREATE TABLE src (key STRING, value STRING) STORED AS TEXTFILE;
+CREATE TABLE src (key STRING COMMENT 'default', value STRING COMMENT 'default') STORED AS TEXTFILE;
LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/kv1.txt" INTO TABLE src;
@@ -12,7 +12,7 @@ LOAD DATA LOCAL INPATH "${hiveconf:test.
--
DROP TABLE IF EXISTS src1;
-CREATE TABLE src1 (key STRING, value STRING) STORED AS TEXTFILE;
+CREATE TABLE src1 (key STRING COMMENT 'default', value STRING COMMENT 'default') STORED AS TEXTFILE;
LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/kv3.txt" INTO TABLE src1;
@@ -21,7 +21,7 @@ LOAD DATA LOCAL INPATH "${hiveconf:test.
--
DROP TABLE IF EXISTS src_json;
-CREATE TABLE src_json (json STRING) STORED AS TEXTFILE;
+CREATE TABLE src_json (json STRING COMMENT 'default') STORED AS TEXTFILE;
LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/json.txt" INTO TABLE src_json;
@@ -31,7 +31,7 @@ LOAD DATA LOCAL INPATH "${hiveconf:test.
--
DROP TABLE IF EXISTS src_sequencefile;
-CREATE TABLE src_sequencefile (key STRING, value STRING) STORED AS SEQUENCEFILE;
+CREATE TABLE src_sequencefile (key STRING COMMENT 'default', value STRING COMMENT 'default') STORED AS SEQUENCEFILE;
LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/kv1.seq" INTO TABLE src_sequencefile;
@@ -45,7 +45,7 @@ CREATE TABLE src_thrift
ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.thrift.ThriftDeserializer'
WITH SERDEPROPERTIES (
'serialization.class' = 'org.apache.hadoop.hive.serde2.thrift.test.Complex',
- 'serialization.format' = 'com.facebook.thrift.protocol.TBinaryProtocol')
+ 'serialization.format' = 'org.apache.thrift.protocol.TBinaryProtocol')
STORED AS SEQUENCEFILE;
LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/complex.seq" INTO TABLE src_thrift;
@@ -75,6 +75,8 @@ STORED AS TEXTFILE;
LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/srcbucket20.txt" INTO TABLE srcbucket2;
LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/srcbucket21.txt" INTO TABLE srcbucket2;
+LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/srcbucket22.txt" INTO TABLE srcbucket2;
+LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/srcbucket23.txt" INTO TABLE srcbucket2;
--
@@ -82,7 +84,7 @@ LOAD DATA LOCAL INPATH "${hiveconf:test.
--
DROP TABLE IF EXISTS srcpart;
-CREATE TABLE srcpart (key STRING, value STRING)
+CREATE TABLE srcpart (key STRING COMMENT 'default', value STRING COMMENT 'default')
PARTITIONED BY (ds STRING, hr STRING)
STORED AS TEXTFILE;
@@ -99,20 +101,46 @@ LOAD DATA LOCAL INPATH "${hiveconf:test.
OVERWRITE INTO TABLE srcpart PARTITION (ds="2008-04-09", hr="12");
+--
+-- Table alltypesorc
+--
+DROP TABLE IF EXISTS alltypesorc;
+CREATE TABLE alltypesorc(
+ ctinyint TINYINT,
+ csmallint SMALLINT,
+ cint INT,
+ cbigint BIGINT,
+ cfloat FLOAT,
+ cdouble DOUBLE,
+ cstring1 STRING,
+ cstring2 STRING,
+ ctimestamp1 TIMESTAMP,
+ ctimestamp2 TIMESTAMP,
+ cboolean1 BOOLEAN,
+ cboolean2 BOOLEAN)
+ STORED AS ORC;
+
+LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/alltypesorc"
+OVERWRITE INTO TABLE alltypesorc;
+
+
+--
+-- Table primitives
+--
DROP TABLE IF EXISTS primitives;
CREATE TABLE primitives (
- id INT,
- bool_col BOOLEAN,
- tinyint_col TINYINT,
- smallint_col SMALLINT,
- int_col INT,
- bigint_col BIGINT,
- float_col FLOAT,
- double_col DOUBLE,
- date_string_col STRING,
- string_col STRING,
- timestamp_col TIMESTAMP)
-PARTITIONED BY (year INT, month INT)
+ id INT COMMENT 'default',
+ bool_col BOOLEAN COMMENT 'default',
+ tinyint_col TINYINT COMMENT 'default',
+ smallint_col SMALLINT COMMENT 'default',
+ int_col INT COMMENT 'default',
+ bigint_col BIGINT COMMENT 'default',
+ float_col FLOAT COMMENT 'default',
+ double_col DOUBLE COMMENT 'default',
+ date_string_col STRING COMMENT 'default',
+ string_col STRING COMMENT 'default',
+ timestamp_col TIMESTAMP COMMENT 'default')
+PARTITIONED BY (year INT COMMENT 'default', month INT COMMENT 'default')
ROW FORMAT DELIMITED
FIELDS TERMINATED BY ','
ESCAPED BY '\\'
@@ -130,3 +158,60 @@ OVERWRITE INTO TABLE primitives PARTITIO
LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/types/primitives/090401.txt"
OVERWRITE INTO TABLE primitives PARTITION(year=2009, month=4);
+--
+-- Function qtest_get_java_boolean
+--
+DROP FUNCTION IF EXISTS qtest_get_java_boolean;
+CREATE FUNCTION qtest_get_java_boolean AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFTestGetJavaBoolean';
+
+--
+-- Table dest1
+--
+DROP TABLE IF EXISTS dest1;
+
+CREATE TABLE dest1 (key STRING COMMENT 'default', value STRING COMMENT 'default')
+STORED AS
+INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat'
+OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat';
+
+--
+-- Table dest2
+--
+DROP TABLE IF EXISTS dest2;
+
+CREATE TABLE dest2 (key STRING COMMENT 'default', value STRING COMMENT 'default')
+STORED AS
+INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat'
+OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat';
+
+--
+-- Table dest3
+--
+DROP TABLE IF EXISTS dest3;
+
+CREATE TABLE dest3 (key STRING COMMENT 'default', value STRING COMMENT 'default')
+PARTITIONED BY (ds STRING, hr STRING)
+STORED AS
+INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat'
+OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat';
+ALTER TABLE dest3 ADD PARTITION (ds='2008-04-08',hr='12');
+
+--
+-- Table dest4
+--
+DROP TABLE IF EXISTS dest4;
+
+CREATE TABLE dest4 (key STRING COMMENT 'default', value STRING COMMENT 'default')
+STORED AS
+INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat'
+OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat';
+
+--
+-- Table dest4_sequencefile
+--
+DROP TABLE IF EXISTS dest4_sequencefile;
+
+CREATE TABLE dest4_sequencefile (key STRING COMMENT 'default', value STRING COMMENT 'default')
+STORED AS
+INPUTFORMAT 'org.apache.hadoop.mapred.SequenceFileInputFormat'
+OUTPUTFORMAT 'org.apache.hadoop.mapred.SequenceFileOutputFormat';
\ No newline at end of file
Modified: hive/trunk/hbase-handler/src/test/templates/TestHBaseCliDriver.vm
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/test/templates/TestHBaseCliDriver.vm?rev=1616913&r1=1616912&r2=1616913&view=diff
==============================================================================
--- hive/trunk/hbase-handler/src/test/templates/TestHBaseCliDriver.vm (original)
+++ hive/trunk/hbase-handler/src/test/templates/TestHBaseCliDriver.vm Sat Aug 9 01:34:24 2014
@@ -44,9 +44,12 @@ public class $className extends TestCase
protected void setUp() {
MiniClusterType miniMR = MiniClusterType.valueForString("$clusterMode");
+ String initScript = "$initScript";
+ String cleanupScript = "$cleanupScript";
try {
- qt = new HBaseQTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR, setup);
+ qt = new HBaseQTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR,
+ setup, initScript, cleanupScript);
} catch (Exception e) {
System.err.println("Exception: " + e.getMessage());
e.printStackTrace();
Modified: hive/trunk/hbase-handler/src/test/templates/TestHBaseNegativeCliDriver.vm
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/test/templates/TestHBaseNegativeCliDriver.vm?rev=1616913&r1=1616912&r2=1616913&view=diff
==============================================================================
--- hive/trunk/hbase-handler/src/test/templates/TestHBaseNegativeCliDriver.vm (original)
+++ hive/trunk/hbase-handler/src/test/templates/TestHBaseNegativeCliDriver.vm Sat Aug 9 01:34:24 2014
@@ -45,9 +45,12 @@ public class $className extends TestCase
protected void setUp() {
MiniClusterType miniMR = MiniClusterType.valueForString("$clusterMode");
+ String initScript = "$initScript";
+ String cleanupScript = "$cleanupScript";
try {
- qt = new HBaseQTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR, setup);
+ qt = new HBaseQTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR,
+ setup, initScript, cleanupScript);
} catch (Exception e) {
System.err.println("Exception: " + e.getMessage());
e.printStackTrace();
Modified: hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java
URL: http://svn.apache.org/viewvc/hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java?rev=1616913&r1=1616912&r2=1616913&view=diff
==============================================================================
--- hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java (original)
+++ hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java Sat Aug 9 01:34:24 2014
@@ -88,7 +88,7 @@ public class TestLocationQueries extends
String hadoopVer, String locationSubdir)
throws Exception
{
- super(outDir, logDir, miniMr, hadoopVer);
+ super(outDir, logDir, miniMr, hadoopVer, "", "");
this.locationSubdir = locationSubdir;
}
}
Modified: hive/trunk/itests/qtest/pom.xml
URL: http://svn.apache.org/viewvc/hive/trunk/itests/qtest/pom.xml?rev=1616913&r1=1616912&r2=1616913&view=diff
==============================================================================
--- hive/trunk/itests/qtest/pom.xml (original)
+++ hive/trunk/itests/qtest/pom.xml Sat Aug 9 01:34:24 2014
@@ -420,7 +420,9 @@
resultsDirectory="${basedir}/${hive.path.to.root}/ql/src/test/results/compiler/" className="TestParse"
logFile="${project.build.directory}/testparsegen.log"
hadoopVersion="${active.hadoop.version}"
- logDirectory="${project.build.directory}/qfile-results/positive/"/>
+ logDirectory="${project.build.directory}/qfile-results/positive/"
+ initScript="q_test_init.sql"
+ cleanupScript="q_test_cleanup.sql"/>
<!-- Negative Parse -->
<qtestgen hiveRootDirectory="${basedir}/${hive.path.to.root}/"
@@ -433,7 +435,9 @@
resultsDirectory="${basedir}/${hive.path.to.root}/ql/src/test/results/compiler/errors/" className="TestParseNegative"
logFile="${project.build.directory}/testparseneggen.log"
hadoopVersion="${active.hadoop.version}"
- logDirectory="${project.build.directory}/qfile-results/negative/"/>
+ logDirectory="${project.build.directory}/qfile-results/negative/"
+ initScript="q_test_init.sql"
+ cleanupScript="q_test_cleanup.sql"/>
<!-- Cli -->
<qtestgen hiveRootDirectory="${basedir}/${hive.path.to.root}/"
@@ -448,7 +452,9 @@
resultsDirectory="${basedir}/${hive.path.to.root}/ql/src/test/results/clientpositive/" className="TestCliDriver"
logFile="${project.build.directory}/testclidrivergen.log"
logDirectory="${project.build.directory}/qfile-results/clientpositive/"
- hadoopVersion="${active.hadoop.version}"/>
+ hadoopVersion="${active.hadoop.version}"
+ initScript="q_test_init.sql"
+ cleanupScript="q_test_cleanup.sql"/>
<!-- Negative Cli -->
<qtestgen hiveRootDirectory="${basedir}/${hive.path.to.root}/"
@@ -463,7 +469,9 @@
resultsDirectory="${basedir}/${hive.path.to.root}/ql/src/test/results/clientnegative/" className="TestNegativeCliDriver"
logFile="${project.build.directory}/testnegativeclidrivergen.log"
logDirectory="${project.build.directory}/qfile-results/clientnegative/"
- hadoopVersion="${active.hadoop.version}"/>
+ hadoopVersion="${active.hadoop.version}"
+ initScript="q_test_init.sql"
+ cleanupScript="q_test_cleanup.sql"/>
<!-- Compare Cli -->
<qtestgen hiveRootDirectory="${basedir}/${hive.path.to.root}/"
@@ -477,7 +485,9 @@
className="TestCompareCliDriver"
logFile="${project.build.directory}/testcompareclidrivergen.log"
logDirectory="${project.build.directory}/qfile-results/clientcompare/"
- hadoopVersion="${active.hadoop.version}"/>
+ hadoopVersion="${active.hadoop.version}"
+ initScript="q_test_init.sql"
+ cleanupScript="q_test_cleanup.sql"/>
<!-- Minimr -->
<qtestgen hiveRootDirectory="${basedir}/${hive.path.to.root}/"
@@ -493,7 +503,8 @@
logFile="${project.build.directory}/testminimrclidrivergen.log"
logDirectory="${project.build.directory}/qfile-results/clientpositive/"
hadoopVersion="${active.hadoop.version}"
- />
+ initScript="q_test_init.sql"
+ cleanupScript="q_test_cleanup.sql"/>
<if>
<equals arg1="${active.hadoop.version}" arg2="${hadoop-23.version}"/>
@@ -513,7 +524,8 @@
logFile="${project.build.directory}/testminitezclidrivergen.log"
logDirectory="${project.build.directory}/qfile-results/clientpositive/"
hadoopVersion="${active.hadoop.version}"
- />
+ initScript="q_test_init.sql"
+ cleanupScript="q_test_cleanup.sql"/>
</then>
<else>
</else>
@@ -533,7 +545,8 @@
logFile="${project.build.directory}/testnegativeminimrclidrivergen.log"
logDirectory="${project.build.directory}/qfile-results/clientnegative/"
hadoopVersion="${hadoopVersion}"
- />
+ initScript="q_test_init.sql"
+ cleanupScript="q_test_cleanup.sql"/>
<!-- HBase Positive -->
<qtestgen hiveRootDirectory="${basedir}/${hive.path.to.root}/"
@@ -545,7 +558,9 @@
clusterMode="${clustermode}"
resultsDirectory="${basedir}/${hive.path.to.root}/hbase-handler/src/test/results/positive/" className="TestHBaseCliDriver"
logFile="${project.build.directory}/testhbaseclidrivergen.log"
- logDirectory="${project.build.directory}/qfile-results/hbase-handler/positive/"/>
+ logDirectory="${project.build.directory}/qfile-results/hbase-handler/positive/"
+ initScript="q_test_init.sql"
+ cleanupScript="q_test_cleanup.sql"/>
<!-- HBase Minimr -->
<qtestgen hiveRootDirectory="${basedir}/${hive.path.to.root}/"
@@ -557,7 +572,9 @@
clusterMode="miniMR"
resultsDirectory="${basedir}/${hive.path.to.root}/hbase-handler/src/test/results/positive/" className="TestHBaseMinimrCliDriver"
logFile="${project.build.directory}/testhbaseminimrclidrivergen.log"
- logDirectory="${project.build.directory}/qfile-results/hbase-handler/minimrpositive/"/>
+ logDirectory="${project.build.directory}/qfile-results/hbase-handler/minimrpositive/"
+ initScript="q_test_init.sql"
+ cleanupScript="q_test_cleanup.sql"/>
<!-- HBase Negative -->
<qtestgen hiveRootDirectory="${basedir}/${hive.path.to.root}/"
@@ -569,7 +586,9 @@
clusterMode="${clustermode}"
resultsDirectory="${basedir}/${hive.path.to.root}/hbase-handler/src/test/results/negative/" className="TestHBaseNegativeCliDriver"
logFile="${project.build.directory}/testhbasenegativeclidrivergen.log"
- logDirectory="${project.build.directory}/qfile-results/hbase-handler/negative"/>
+ logDirectory="${project.build.directory}/qfile-results/hbase-handler/negative"
+ initScript="q_test_init.sql"
+ cleanupScript="q_test_cleanup.sql"/>
<!-- Beeline -->
@@ -588,7 +607,7 @@
resultsDirectory="${basedir}/${hive.path.to.root}/ql/src/test/results/clientpositive/" className="TestBeeLineDriver"
logFile="${project.build.directory}/testbeelinedrivergen.log"
logDirectory="${project.build.directory}/qfile-results/beelinepositive/"
- hadoopVersion="${hadoopVersion}" />
+ hadoopVersion="${hadoopVersion}"/>
</then>
</if>
@@ -606,7 +625,8 @@
logFile="${project.build.directory}/testcontribclidrivergen.log"
logDirectory="${project.build.directory}/qfile-results/contribclientpositive"
hadoopVersion="${hadoopVersion}"
- />
+ initScript="q_test_init.sql"
+ cleanupScript="q_test_cleanup.sql"/>
<qtestgen hiveRootDirectory="${basedir}/${hive.path.to.root}/"
outputDirectory="${project.build.directory}/generated-test-sources/java/org/apache/hadoop/hive/cli"
@@ -617,7 +637,9 @@
runDisabled="${run_disabled}"
resultsDirectory="${basedir}/${hive.path.to.root}/contrib/src/test/results/clientnegative/" className="TestContribNegativeCliDriver"
logFile="${project.build.directory}/testcontribnegclidrivergen.log"
- logDirectory="${project.build.directory}/qfile-results/contribclientnegative"/>
+ logDirectory="${project.build.directory}/qfile-results/contribclientnegative"
+ initScript="q_test_init.sql"
+ cleanupScript="q_test_cleanup.sql"/>
</target>
Modified: hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java
URL: http://svn.apache.org/viewvc/hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java?rev=1616913&r1=1616912&r2=1616913&view=diff
==============================================================================
--- hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java (original)
+++ hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java Sat Aug 9 01:34:24 2014
@@ -40,10 +40,11 @@ public class HBaseQTestUtil extends QTes
private final HConnection conn;
public HBaseQTestUtil(
- String outDir, String logDir, MiniClusterType miniMr, HBaseTestSetup setup)
+ String outDir, String logDir, MiniClusterType miniMr, HBaseTestSetup setup,
+ String initScript, String cleanupScript)
throws Exception {
- super(outDir, logDir, miniMr, null);
+ super(outDir, logDir, miniMr, null, initScript, cleanupScript);
setup.preTest(conf);
this.conn = setup.getConnection();
super.init();
Modified: hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
URL: http://svn.apache.org/viewvc/hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java?rev=1616913&r1=1616912&r2=1616913&view=diff
==============================================================================
--- hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java (original)
+++ hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java Sat Aug 9 01:34:24 2014
@@ -38,7 +38,6 @@ import java.io.OutputStreamWriter;
import java.io.PrintStream;
import java.io.Serializable;
import java.io.StringWriter;
-import java.io.UnsupportedEncodingException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
@@ -115,6 +114,8 @@ public class QTestUtil {
public static final String UTF_8 = "UTF-8";
private static final Log LOG = LogFactory.getLog("QTestUtil");
+ private final String defaultInitScript = "q_test_init.sql";
+ private final String defaultCleanupScript = "q_test_cleanup.sql";
private String testWarehouse;
private final String testFiles;
@@ -142,6 +143,10 @@ public class QTestUtil {
private boolean miniMr = false;
private String hadoopVer = null;
private QTestSetup setup = null;
+ private boolean isSessionStateStarted = false;
+
+ private String initScript;
+ private String cleanupScript;
static {
for (String srcTable : System.getProperty("test.src.tables", "").trim().split(",")) {
@@ -225,8 +230,9 @@ public class QTestUtil {
}
}
- public QTestUtil(String outDir, String logDir) throws Exception {
- this(outDir, logDir, MiniClusterType.none, null, "0.20");
+ public QTestUtil(String outDir, String logDir, String initScript, String cleanupScript) throws
+ Exception {
+ this(outDir, logDir, MiniClusterType.none, null, "0.20", initScript, cleanupScript);
}
public String getOutputDirectory() {
@@ -297,13 +303,14 @@ public class QTestUtil {
}
}
- public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, String hadoopVer)
+ public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, String hadoopVer,
+ String initScript, String cleanupScript)
throws Exception {
- this(outDir, logDir, clusterType, null, hadoopVer);
+ this(outDir, logDir, clusterType, null, hadoopVer, initScript, cleanupScript);
}
public QTestUtil(String outDir, String logDir, MiniClusterType clusterType,
- String confDir, String hadoopVer)
+ String confDir, String hadoopVer, String initScript, String cleanupScript)
throws Exception {
this.outDir = outDir;
this.logDir = logDir;
@@ -354,6 +361,20 @@ public class QTestUtil {
testFiles = dataDir;
+ // Use the current directory if it is not specified
+ String scriptsDir = conf.get("test.data.scripts");
+ if (scriptsDir == null) {
+ scriptsDir = new File(".").getAbsolutePath() + "/data/scripts";
+ }
+ if (initScript.isEmpty()) {
+ initScript = defaultInitScript;
+ }
+ if (cleanupScript.isEmpty()) {
+ cleanupScript = defaultCleanupScript;
+ }
+ this.initScript = scriptsDir + "/" + initScript;
+ this.cleanupScript = scriptsDir + "/" + cleanupScript;
+
overWrite = "true".equalsIgnoreCase(System.getProperty("test.output.overwrite"));
setup = new QTestSetup();
@@ -593,14 +614,15 @@ public class QTestUtil {
}
public void cleanUp() throws Exception {
- // Drop any tables that remain due to unsuccessful runs
- for (String s : new String[] {"src", "src1", "src_json", "src_thrift",
- "src_sequencefile", "srcpart", "srcbucket", "srcbucket2", "dest1",
- "dest2", "dest3", "dest4", "dest4_sequencefile", "dest_j1", "dest_j2",
- "dest_g1", "dest_g2", "fetchtask_ioexception",
- AllVectorTypesRecord.TABLE_NAME}) {
- db.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, s);
+ if(!isSessionStateStarted) {
+ startSessionState();
}
+ String cleanupCommands = readEntireFileIntoString(new File(cleanupScript));
+ LOG.info("Cleanup (" + cleanupScript + "):\n" + cleanupCommands);
+ if(cliDriver == null) {
+ cliDriver = new CliDriver();
+ }
+ cliDriver.processLine(cleanupCommands);
// delete any contents in the warehouse dir
Path p = new Path(testWarehouse);
@@ -653,119 +675,20 @@ public class QTestUtil {
}
public void createSources() throws Exception {
-
- startSessionState();
+ if(!isSessionStateStarted) {
+ startSessionState();
+ }
conf.setBoolean("hive.test.init.phase", true);
- // Create a bunch of tables with columns key and value
- LinkedList<String> cols = new LinkedList<String>();
- cols.add("key");
- cols.add("value");
-
- LinkedList<String> part_cols = new LinkedList<String>();
- part_cols.add("ds");
- part_cols.add("hr");
- db.createTable("srcpart", cols, part_cols, TextInputFormat.class,
- IgnoreKeyTextOutputFormat.class);
-
- Path fpath;
- HashMap<String, String> part_spec = new HashMap<String, String>();
- for (String ds : new String[] {"2008-04-08", "2008-04-09"}) {
- for (String hr : new String[] {"11", "12"}) {
- part_spec.clear();
- part_spec.put("ds", ds);
- part_spec.put("hr", hr);
- // System.out.println("Loading partition with spec: " + part_spec);
- // db.createPartition(srcpart, part_spec);
- fpath = new Path(testFiles, "kv1.txt");
- // db.loadPartition(fpath, srcpart.getName(), part_spec, true);
- runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toUri().getPath()
- + "' OVERWRITE INTO TABLE srcpart PARTITION (ds='" + ds + "',hr='"
- + hr + "')");
- }
- }
- ArrayList<String> bucketCols = new ArrayList<String>();
- bucketCols.add("key");
- runCreateTableCmd("CREATE TABLE srcbucket(key int, value string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE");
- // db.createTable("srcbucket", cols, null, TextInputFormat.class,
- // IgnoreKeyTextOutputFormat.class, 2, bucketCols);
- for (String fname : new String[] {"srcbucket0.txt", "srcbucket1.txt"}) {
- fpath = new Path(testFiles, fname);
- runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toUri().getPath()
- + "' INTO TABLE srcbucket");
- }
-
- runCreateTableCmd("CREATE TABLE srcbucket2(key int, value string) "
- + "CLUSTERED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE");
- // db.createTable("srcbucket", cols, null, TextInputFormat.class,
- // IgnoreKeyTextOutputFormat.class, 2, bucketCols);
- for (String fname : new String[] {"srcbucket20.txt", "srcbucket21.txt",
- "srcbucket22.txt", "srcbucket23.txt"}) {
- fpath = new Path(testFiles, fname);
- runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toUri().getPath()
- + "' INTO TABLE srcbucket2");
- }
-
- for (String tname : new String[] {"src", "src1"}) {
- db.createTable(tname, cols, null, TextInputFormat.class,
- IgnoreKeyTextOutputFormat.class);
- }
- db.createTable("src_sequencefile", cols, null,
- SequenceFileInputFormat.class, SequenceFileOutputFormat.class);
-
- Table srcThrift =
- new Table(SessionState.get().getCurrentDatabase(), "src_thrift");
- srcThrift.setInputFormatClass(SequenceFileInputFormat.class.getName());
- srcThrift.setOutputFormatClass(SequenceFileOutputFormat.class.getName());
- srcThrift.setSerializationLib(ThriftDeserializer.class.getName());
- srcThrift.setSerdeParam(serdeConstants.SERIALIZATION_CLASS, Complex.class
- .getName());
- srcThrift.setSerdeParam(serdeConstants.SERIALIZATION_FORMAT,
- TBinaryProtocol.class.getName());
- db.createTable(srcThrift);
-
- LinkedList<String> json_cols = new LinkedList<String>();
- json_cols.add("json");
- db.createTable("src_json", json_cols, null, TextInputFormat.class,
- IgnoreKeyTextOutputFormat.class);
-
- // load the input data into the src table
- fpath = new Path(testFiles, "kv1.txt");
- runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toUri().getPath() + "' INTO TABLE src");
-
- // load the input data into the src table
- fpath = new Path(testFiles, "kv3.txt");
- runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toUri().getPath() + "' INTO TABLE src1");
-
- // load the input data into the src_sequencefile table
- fpath = new Path(testFiles, "kv1.seq");
- runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toUri().getPath()
- + "' INTO TABLE src_sequencefile");
-
- // load the input data into the src_thrift table
- fpath = new Path(testFiles, "complex.seq");
- runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toUri().getPath()
- + "' INTO TABLE src_thrift");
-
- // load the json data into the src_json table
- fpath = new Path(testFiles, "json.txt");
- runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toUri().getPath()
- + "' INTO TABLE src_json");
-
- FileSystem localFs = FileSystem.getLocal(conf);
- // create and load data into orc table
- fpath = new Path(testFiles, AllVectorTypesRecord.TABLE_NAME);
-
- runCreateTableCmd(AllVectorTypesRecord.TABLE_CREATE_COMMAND);
- runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toUri().getPath()
- + "' INTO TABLE "+AllVectorTypesRecord.TABLE_NAME);
-
- runCmd("DROP FUNCTION IF EXISTS qtest_get_java_boolean ");
- runCmd("CREATE FUNCTION qtest_get_java_boolean "
- + " AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFTestGetJavaBoolean'");
+ String initCommands = readEntireFileIntoString(new File(this.initScript));
+ LOG.info("Initial setup (" + initScript + "):\n" + initCommands);
+ if(cliDriver == null) {
+ cliDriver = new CliDriver();
+ }
+ cliDriver.processLine("set test.data.dir=" + testFiles + ";");
+ cliDriver.processLine(initCommands);
conf.setBoolean("hive.test.init.phase", false);
-
}
public void init() throws Exception {
@@ -786,33 +709,7 @@ public class QTestUtil {
public void init(String tname) throws Exception {
cleanUp();
createSources();
-
- LinkedList<String> cols = new LinkedList<String>();
- cols.add("key");
- cols.add("value");
-
- LinkedList<String> part_cols = new LinkedList<String>();
- part_cols.add("ds");
- part_cols.add("hr");
-
- db.createTable("dest1", cols, null, TextInputFormat.class,
- IgnoreKeyTextOutputFormat.class);
- db.createTable("dest2", cols, null, TextInputFormat.class,
- IgnoreKeyTextOutputFormat.class);
-
- db.createTable("dest3", cols, part_cols, TextInputFormat.class,
- IgnoreKeyTextOutputFormat.class);
- Table dest3 = db.getTable("dest3");
-
- HashMap<String, String> part_spec = new HashMap<String, String>();
- part_spec.put("ds", "2008-04-08");
- part_spec.put("hr", "12");
- db.createPartition(dest3, part_spec);
-
- db.createTable("dest4", cols, null, TextInputFormat.class,
- IgnoreKeyTextOutputFormat.class);
- db.createTable("dest4_sequencefile", cols, null,
- SequenceFileInputFormat.class, SequenceFileOutputFormat.class);
+ cliDriver.processCmd("set hive.cli.print.header=true;");
}
public void cliInit(String tname) throws Exception {
@@ -866,23 +763,38 @@ public class QTestUtil {
SessionState.start(ss);
cliDriver = new CliDriver();
+
if (tname.equals("init_file.q")) {
ss.initFiles.add("../../data/scripts/test_init_file.sql");
}
cliDriver.processInitFiles(ss);
+
return outf.getAbsolutePath();
}
private CliSessionState startSessionState()
- throws FileNotFoundException, UnsupportedEncodingException {
+ throws IOException {
HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER,
"org.apache.hadoop.hive.ql.security.DummyAuthenticator");
CliSessionState ss = new CliSessionState(conf);
assert ss != null;
+ ss.in = System.in;
+ ss.out = System.out;
+ ss.err = System.out;
+ SessionState oldSs = SessionState.get();
+ if (oldSs != null && clusterType == MiniClusterType.tez) {
+ oldSs.close();
+ }
+ if (oldSs != null && oldSs.out != null && oldSs.out != System.out) {
+ oldSs.out.close();
+ }
SessionState.start(ss);
+
+ isSessionStateStarted = true;
+
return ss;
}
@@ -1571,7 +1483,7 @@ public class QTestUtil {
{
QTestUtil[] qt = new QTestUtil[qfiles.length];
for (int i = 0; i < qfiles.length; i++) {
- qt[i] = new QTestUtil(resDir, logDir, MiniClusterType.none, null, "0.20");
+ qt[i] = new QTestUtil(resDir, logDir, MiniClusterType.none, null, "0.20", "", "");
qt[i].addFile(qfiles[i]);
qt[i].clearTestSideEffects();
}
Modified: hive/trunk/ql/src/test/templates/TestCliDriver.vm
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/templates/TestCliDriver.vm?rev=1616913&r1=1616912&r2=1616913&view=diff
==============================================================================
--- hive/trunk/ql/src/test/templates/TestCliDriver.vm (original)
+++ hive/trunk/ql/src/test/templates/TestCliDriver.vm Sat Aug 9 01:34:24 2014
@@ -37,9 +37,12 @@ public class $className extends TestCase
MiniClusterType miniMR = MiniClusterType.valueForString("$clusterMode");
String hiveConfDir = "$hiveConfDir";
+ String initScript = "$initScript";
+ String cleanupScript = "$cleanupScript";
try {
String hadoopVer = "$hadoopVersion";
- qt = new QTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR, hiveConfDir, hadoopVer);
+ qt = new QTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR,
+ hiveConfDir, hadoopVer, initScript, cleanupScript);
// do a one time initialization
qt.cleanUp();
Modified: hive/trunk/ql/src/test/templates/TestCompareCliDriver.vm
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/templates/TestCompareCliDriver.vm?rev=1616913&r1=1616912&r2=1616913&view=diff
==============================================================================
--- hive/trunk/ql/src/test/templates/TestCompareCliDriver.vm (original)
+++ hive/trunk/ql/src/test/templates/TestCompareCliDriver.vm Sat Aug 9 01:34:24 2014
@@ -38,9 +38,12 @@ public class $className extends TestCase
MiniClusterType miniMR = MiniClusterType.valueForString("$clusterMode");
String hiveConfDir = "$hiveConfDir";
+ String initScript = "$initScript";
+ String cleanupScript = "$cleanupScript";
try {
String hadoopVer = "$hadoopVersion";
- qt = new QTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR, hiveConfDir, hadoopVer);
+ qt = new QTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR,
+ hiveConfDir, hadoopVer, initScript, cleanupScript);
// do a one time initialization
qt.cleanUp();
Modified: hive/trunk/ql/src/test/templates/TestNegativeCliDriver.vm
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/templates/TestNegativeCliDriver.vm?rev=1616913&r1=1616912&r2=1616913&view=diff
==============================================================================
--- hive/trunk/ql/src/test/templates/TestNegativeCliDriver.vm (original)
+++ hive/trunk/ql/src/test/templates/TestNegativeCliDriver.vm Sat Aug 9 01:34:24 2014
@@ -35,10 +35,13 @@ public class $className extends TestCase
static {
MiniClusterType miniMR = MiniClusterType.valueForString("$clusterMode");
+ String initScript = "$initScript";
+ String cleanupScript = "$cleanupScript";
try {
String hadoopVer = "$hadoopVersion";
- qt = new QTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR, hadoopVer);
+ qt = new QTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR, hadoopVer,
+ initScript, cleanupScript);
// do a one time initialization
qt.cleanUp();
qt.createSources();
Modified: hive/trunk/ql/src/test/templates/TestParse.vm
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/templates/TestParse.vm?rev=1616913&r1=1616912&r2=1616913&view=diff
==============================================================================
--- hive/trunk/ql/src/test/templates/TestParse.vm (original)
+++ hive/trunk/ql/src/test/templates/TestParse.vm Sat Aug 9 01:34:24 2014
@@ -35,10 +35,13 @@ public class $className extends TestCase
static {
MiniClusterType miniMR = MiniClusterType.valueForString("$clusterMode");
+ String initScript = "$initScript";
+ String cleanupScript = "$cleanupScript";
try {
String hadoopVer = "$hadoopVersion";
- qt = new QTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR, hadoopVer);
+ qt = new QTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR, hadoopVer,
+ initScript, cleanupScript);
qt.init(null);
} catch (Exception e) {
System.err.println("Exception: " + e.getMessage());
Modified: hive/trunk/ql/src/test/templates/TestParseNegative.vm
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/templates/TestParseNegative.vm?rev=1616913&r1=1616912&r2=1616913&view=diff
==============================================================================
--- hive/trunk/ql/src/test/templates/TestParseNegative.vm (original)
+++ hive/trunk/ql/src/test/templates/TestParseNegative.vm Sat Aug 9 01:34:24 2014
@@ -36,10 +36,13 @@ public class $className extends TestCase
static {
MiniClusterType miniMR = MiniClusterType.valueForString("$clusterMode");
+ String initScript = "$initScript";
+ String cleanupScript = "$cleanupScript";
try {
String hadoopVer = "$hadoopVersion";
- qt = new QTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR, hadoopVer);
+ qt = new QTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR, hadoopVer,
+ initScript, cleanupScript);
} catch (Exception e) {
System.err.println("Exception: " + e.getMessage());
e.printStackTrace();