You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by nz...@apache.org on 2010/07/21 05:42:17 UTC
svn commit: r966076 - in /hadoop/hive/trunk: ./ data/conf/
ql/src/test/org/apache/hadoop/hive/ql/
ql/src/test/org/apache/hadoop/hive/ql/hooks/
ql/src/test/results/clientnegative/ ql/src/test/templates/
Author: nzhang
Date: Wed Jul 21 03:42:16 2010
New Revision: 966076
URL: http://svn.apache.org/viewvc?rev=966076&view=rev
Log:
HIVE-1464 improve test query performance (Joydeep Sen Sarma via Ning Zhang)
Added:
hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/hooks/EnforceReadOnlyTables.java
Modified:
hadoop/hive/trunk/CHANGES.txt
hadoop/hive/trunk/data/conf/hive-site.xml
hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java
hadoop/hive/trunk/ql/src/test/results/clientnegative/invalid_t_alter2.q.out
hadoop/hive/trunk/ql/src/test/results/clientnegative/load_wrong_fileformat_txt_seq.q.out
hadoop/hive/trunk/ql/src/test/templates/TestCliDriver.vm
hadoop/hive/trunk/ql/src/test/templates/TestNegativeCliDriver.vm
Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=966076&r1=966075&r2=966076&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Wed Jul 21 03:42:16 2010
@@ -71,6 +71,11 @@ Trunk - Unreleased
HIVE-1455. lateral view does not work with column pruning
(Paul Yang via He Yongqiang)
+ TESTS
+
+ HIVE-1464. improve test query performance
+ (Joydeep Sen Sarma via Ning Zhang)
+
Release 0.6.0 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/hive/trunk/data/conf/hive-site.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/data/conf/hive-site.xml?rev=966076&r1=966075&r2=966076&view=diff
==============================================================================
--- hadoop/hive/trunk/data/conf/hive-site.xml (original)
+++ hadoop/hive/trunk/data/conf/hive-site.xml Wed Jul 21 03:42:16 2010
@@ -129,7 +129,7 @@
<property>
<name>hive.exec.pre.hooks</name>
- <value>org.apache.hadoop.hive.ql.hooks.PreExecutePrinter</value>
+ <value>org.apache.hadoop.hive.ql.hooks.PreExecutePrinter, org.apache.hadoop.hive.ql.hooks.EnforceReadOnlyTables</value>
<description>Pre Execute Hook for Tests</description>
</property>
Modified: hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java?rev=966076&r1=966075&r2=966076&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java (original)
+++ hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java Wed Jul 21 03:42:16 2010
@@ -28,6 +28,7 @@ import java.io.PrintStream;
import java.io.Serializable;
import java.net.URI;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Deque;
import java.util.HashMap;
import java.util.HashSet;
@@ -84,11 +85,15 @@ public class QTestUtil {
private final String logDir;
private final TreeMap<String, String> qMap;
private final Set<String> qSkipSet;
- private final LinkedList<String> srcTables;
+ public static final HashSet<String> srcTables = new HashSet<String>
+ (Arrays.asList(new String [] {
+ "src", "src1", "srcbucket", "srcbucket2", "src_json", "src_thrift",
+ "src_sequencefile", "srcpart"
+ }));
private ParseDriver pd;
private Hive db;
- protected final HiveConf conf;
+ protected HiveConf conf;
private Driver drv;
private SemanticAnalyzer sem;
private FileSystem fs;
@@ -184,6 +189,18 @@ public class QTestUtil {
return null;
}
+ public void initConf() {
+ if (miniMr) {
+ String fsName = conf.get("fs.default.name");
+ assert fsName != null;
+ // hive.metastore.warehouse.dir needs to be set relative to the jobtracker
+ conf.set("hive.metastore.warehouse.dir", fsName
+ .concat("/build/ql/test/data/warehouse/"));
+ conf.set("mapred.job.tracker", "localhost:" + mr.getJobTrackerPort());
+ }
+
+ }
+
public QTestUtil(String outDir, String logDir, boolean miniMr, String hadoopVer) throws Exception {
this.outDir = outDir;
this.logDir = logDir;
@@ -197,17 +214,10 @@ public class QTestUtil {
dfs = ShimLoader.getHadoopShims().getMiniDfs(conf, 4, true, null);
FileSystem fs = dfs.getFileSystem();
mr = new MiniMRCluster(4, fs.getUri().toString(), 1);
-
- // hive.metastore.warehouse.dir needs to be set relative to the jobtracker
- String fsName = conf.get("fs.default.name");
- assert fsName != null;
- conf.set("hive.metastore.warehouse.dir", fsName
- .concat("/build/ql/test/data/warehouse/"));
-
- conf.set("mapred.job.tracker", "localhost:" + mr.getJobTrackerPort());
}
- // System.out.println(conf.toString());
+ initConf();
+
testFiles = conf.get("test.data.files").replace('\\', '/')
.replace("c:", "");
@@ -217,8 +227,6 @@ public class QTestUtil {
overWrite = true;
}
- srcTables = new LinkedList<String>();
-
init();
}
@@ -290,6 +298,22 @@ public class QTestUtil {
dis.close();
}
+ /**
+ * Clear out any side effects of running tests
+ */
+ public void clearTestSideEffects () throws Exception {
+ // delete any tables other than the source tables
+ for (String s: db.getAllTables()) {
+ if (!srcTables.contains(s))
+ db.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, s);
+ }
+ // allocate and initialize a new conf since a test can
+ // modify conf by using 'set' commands
+ conf = new HiveConf (Driver.class);
+ initConf();
+ }
+
+
public void cleanUp() throws Exception {
String warehousePath = ((new URI(testWarehouse)).getPath());
// Drop any tables that remain due to unsuccessful runs
@@ -329,8 +353,7 @@ public class QTestUtil {
}
public void createSources() throws Exception {
- // Next create the three tables src, dest1 and dest2 each with two columns
- // key and value
+ // Create a bunch of tables with columns key and value
LinkedList<String> cols = new LinkedList<String>();
cols.add("key");
cols.add("value");
@@ -340,7 +363,6 @@ public class QTestUtil {
part_cols.add("hr");
db.createTable("srcpart", cols, part_cols, TextInputFormat.class,
IgnoreKeyTextOutputFormat.class);
- srcTables.add("srcpart");
Path fpath;
Path newfpath;
@@ -367,7 +389,6 @@ public class QTestUtil {
runCreateTableCmd("CREATE TABLE srcbucket(key int, value string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE");
// db.createTable("srcbucket", cols, null, TextInputFormat.class,
// IgnoreKeyTextOutputFormat.class, 2, bucketCols);
- srcTables.add("srcbucket");
for (String fname : new String[] {"srcbucket0.txt", "srcbucket1.txt"}) {
fpath = new Path(testFiles, fname);
newfpath = new Path(tmppath, fname);
@@ -380,7 +401,6 @@ public class QTestUtil {
+ "CLUSTERED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE");
// db.createTable("srcbucket", cols, null, TextInputFormat.class,
// IgnoreKeyTextOutputFormat.class, 2, bucketCols);
- srcTables.add("srcbucket2");
for (String fname : new String[] {"srcbucket20.txt", "srcbucket21.txt",
"srcbucket22.txt", "srcbucket23.txt"}) {
fpath = new Path(testFiles, fname);
@@ -393,11 +413,9 @@ public class QTestUtil {
for (String tname : new String[] {"src", "src1"}) {
db.createTable(tname, cols, null, TextInputFormat.class,
IgnoreKeyTextOutputFormat.class);
- srcTables.add(tname);
}
db.createTable("src_sequencefile", cols, null,
SequenceFileInputFormat.class, SequenceFileOutputFormat.class);
- srcTables.add("src_sequencefile");
Table srcThrift = new Table("src_thrift");
srcThrift.setInputFormatClass(SequenceFileInputFormat.class.getName());
@@ -408,13 +426,11 @@ public class QTestUtil {
srcThrift.setSerdeParam(Constants.SERIALIZATION_FORMAT,
TBinaryProtocol.class.getName());
db.createTable(srcThrift);
- srcTables.add("src_thrift");
LinkedList<String> json_cols = new LinkedList<String>();
json_cols.add("json");
db.createTable("src_json", json_cols, null, TextInputFormat.class,
IgnoreKeyTextOutputFormat.class);
- srcTables.add("src_json");
// load the input data into the src table
fpath = new Path(testFiles, "kv1.txt");
@@ -512,7 +528,7 @@ public class QTestUtil {
createSources();
}
- CliSessionState ss = new CliSessionState(conf);
+ CliSessionState ss = new CliSessionState(new HiveConf(Driver.class));
assert ss != null;
ss.in = System.in;
Added: hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/hooks/EnforceReadOnlyTables.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/hooks/EnforceReadOnlyTables.java?rev=966076&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/hooks/EnforceReadOnlyTables.java (added)
+++ hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/hooks/EnforceReadOnlyTables.java Wed Jul 21 03:42:16 2010
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.hooks;
+
+import java.util.Set;
+
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.hive.ql.QTestUtil;
+import org.apache.hadoop.hive.ql.metadata.Table;
+
+/**
+ * Implementation of a pre execute hook that prevents modifications
+ * of read-only tables used by the test framework
+ */
+public class EnforceReadOnlyTables implements PreExecute {
+
+ @Override
+ public void run(SessionState sess, Set<ReadEntity> inputs,
+ Set<WriteEntity> outputs, UserGroupInformation ugi)
+ throws Exception {
+
+ for (WriteEntity w: outputs) {
+ if ((w.getTyp() == WriteEntity.Type.TABLE) ||
+ (w.getTyp() == WriteEntity.Type.PARTITION)) {
+ Table t = w.getTable();
+ if (QTestUtil.srcTables.contains(t.getTableName()))
+ throw new RuntimeException ("Cannot overwrite read-only table: " + t.getTableName());
+ }
+ }
+ }
+}
Modified: hadoop/hive/trunk/ql/src/test/results/clientnegative/invalid_t_alter2.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientnegative/invalid_t_alter2.q.out?rev=966076&r1=966075&r2=966076&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientnegative/invalid_t_alter2.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientnegative/invalid_t_alter2.q.out Wed Jul 21 03:42:16 2010
@@ -1,4 +1,6 @@
PREHOOK: query: CREATE TABLE alter_test (d STRING)
PREHOOK: type: CREATETABLE
-FAILED: Error in metadata: AlreadyExistsException(message:Table alter_test already exists)
-FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask
+POSTHOOK: query: CREATE TABLE alter_test (d STRING)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@alter_test
+FAILED: Error in semantic analysis: DATE, DATETIME, and TIMESTAMP types aren't supported yet. Please use STRING instead.
Modified: hadoop/hive/trunk/ql/src/test/results/clientnegative/load_wrong_fileformat_txt_seq.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientnegative/load_wrong_fileformat_txt_seq.q.out?rev=966076&r1=966075&r2=966076&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientnegative/load_wrong_fileformat_txt_seq.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientnegative/load_wrong_fileformat_txt_seq.q.out Wed Jul 21 03:42:16 2010
@@ -8,7 +8,6 @@ POSTHOOK: query: -- test for loading int
DROP TABLE T1
POSTHOOK: type: DROPTABLE
-POSTHOOK: Output: default@t1
PREHOOK: query: CREATE TABLE T1(name STRING) STORED AS TEXTFILE
PREHOOK: type: CREATETABLE
POSTHOOK: query: CREATE TABLE T1(name STRING) STORED AS TEXTFILE
Modified: hadoop/hive/trunk/ql/src/test/templates/TestCliDriver.vm
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/templates/TestCliDriver.vm?rev=966076&r1=966075&r2=966076&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/templates/TestCliDriver.vm (original)
+++ hadoop/hive/trunk/ql/src/test/templates/TestCliDriver.vm Wed Jul 21 03:42:16 2010
@@ -22,27 +22,38 @@ import org.antlr.runtime.tree.*;
public class $className extends TestCase {
- private QTestUtil qt;
+ private static QTestUtil qt;
- public $className(String name) {
- super(name);
- qt = null;
- }
-
- @Override
- protected void setUp() {
+ static {
try {
boolean miniMR = false;
String hadoopVer;
-
if ("$clusterMode".equals("miniMR"))
miniMR = true;
hadoopVer = "$hadoopVersion";
-
qt = new QTestUtil("$resultsDir.getCanonicalPath()", "$logDir.getCanonicalPath()", miniMR, hadoopVer);
+ // do a one time initialization
+ qt.cleanUp();
+ qt.createSources();
+
+ } catch (Exception e) {
+ System.out.println("Exception: " + e.getMessage());
+ e.printStackTrace();
+ System.out.flush();
+ fail("Unexpected exception in static initialization");
}
- catch (Exception e) {
+ }
+
+ public $className(String name) {
+ super(name);
+ }
+
+ @Override
+ protected void setUp() {
+ try {
+ qt.clearTestSideEffects();
+ } catch (Exception e) {
System.out.println("Exception: " + e.getMessage());
e.printStackTrace();
System.out.flush();
@@ -50,10 +61,18 @@ public class $className extends TestCase
}
}
+ /**
+ * Dummy last test. This is only meant to shutdown qt
+ */
+ public void testCliDriver_shutdown() {
+ System.out.println ("Cleaning up " + "$className");
+ }
+
@Override
protected void tearDown() {
try {
- qt.shutdown();
+ if (getName().equals("testCliDriver_shutdown"))
+ qt.shutdown();
}
catch (Exception e) {
System.out.println("Exception: " + e.getMessage());
@@ -71,6 +90,7 @@ public class $className extends TestCase
#set ($tname = $fname.substring(0, $eidx))
suite.addTest(new $className("testCliDriver_$tname"));
#end
+ suite.addTest(new $className("testCliDriver_shutdown"));
return suite;
}
@@ -91,7 +111,7 @@ public class $className extends TestCase
return;
}
- qt.cliInit("$fname");
+ qt.cliInit("$fname", false);
int ecode = qt.executeClient("$fname");
if (ecode != 0) {
fail("Client Execution failed with error code = " + ecode + debugHint);
Modified: hadoop/hive/trunk/ql/src/test/templates/TestNegativeCliDriver.vm
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/templates/TestNegativeCliDriver.vm?rev=966076&r1=966075&r2=966076&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/templates/TestNegativeCliDriver.vm (original)
+++ hadoop/hive/trunk/ql/src/test/templates/TestNegativeCliDriver.vm Wed Jul 21 03:42:16 2010
@@ -15,18 +15,30 @@ import org.antlr.runtime.tree.*;
public class $className extends TestCase {
- private QTestUtil qt;
+ private static QTestUtil qt;
+ static {
+ try {
+ qt = new QTestUtil("$resultsDir.getCanonicalPath()", "$logDir.getCanonicalPath()");
+ // do a one time initialization
+ qt.cleanUp();
+ qt.createSources();
+
+ } catch (Exception e) {
+ System.out.println("Exception: " + e.getMessage());
+ e.printStackTrace();
+ System.out.flush();
+ fail("Unexpected exception in static initialization");
+ }
+ }
public $className(String name) {
super(name);
- qt = null;
}
@Override
protected void setUp() {
try {
- qt = new QTestUtil("$resultsDir.getCanonicalPath()", "$logDir.getCanonicalPath()");
-
+ qt.clearTestSideEffects();
}
catch (Throwable e) {
e.printStackTrace();
@@ -64,7 +76,7 @@ public class $className extends TestCase
return;
}
- qt.cliInit("$fname");
+ qt.cliInit("$fname", false);
int ecode = qt.executeClient("$fname");
if (ecode == 0) {
fail("Client Execution failed with error code = " + ecode