You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by da...@apache.org on 2015/09/22 07:04:12 UTC
[41/50] [abbrv] hive git commit: HIVE-11621 Fix TestMiniTezCliDriver
test failures when HBase Metastore is used (Daniel Dai via gates)
HIVE-11621 Fix TestMiniTezCliDriver test failures when HBase Metastore is used (Daniel Dai via gates)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f014f0da
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f014f0da
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f014f0da
Branch: refs/heads/master
Commit: f014f0da457d1e959cb7da2824f6cf2c5ee0c971
Parents: 4d66206
Author: Alan Gates <ga...@hortonworks.com>
Authored: Fri Aug 28 10:38:49 2015 -0700
Committer: Alan Gates <ga...@hortonworks.com>
Committed: Fri Aug 28 10:38:49 2015 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/ant/QTestGenTask.java | 11 ++
data/conf/tez/hive-site.xml | 9 ++
itests/qtest/pom.xml | 3 +-
.../org/apache/hadoop/hive/ql/QTestUtil.java | 80 +++++-------
.../hadoop/hive/metastore/HiveAlterHandler.java | 22 ++--
.../hive/metastore/hbase/HBaseReadWrite.java | 17 ++-
.../hadoop/hive/metastore/hbase/HBaseStore.java | 127 +++++++++++++++----
.../hadoop/hive/metastore/hbase/HBaseUtils.java | 60 ++++++---
.../hadoop/hive/metastore/hbase/StatsCache.java | 2 +-
.../stats/ColumnStatsAggregatorFactory.java | 9 +-
.../apache/hadoop/hive/ql/stats/StatsUtils.java | 3 +
.../dynpart_sort_opt_vectorization.q | 2 +
.../clientpositive/dynpart_sort_optimization.q | 2 +
.../tez/dynpart_sort_opt_vectorization.q.out | 12 +-
.../tez/dynpart_sort_optimization.q.out | 12 +-
ql/src/test/templates/TestCliDriver.vm | 3 +-
16 files changed, 248 insertions(+), 126 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/f014f0da/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java
----------------------------------------------------------------------
diff --git a/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java b/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java
index 7fffe13..8b1c4fe 100644
--- a/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java
+++ b/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java
@@ -149,6 +149,8 @@ public class QTestGenTask extends Task {
private String cleanupScript;
+ private String useHBaseMetastore;
+
public void setHadoopVersion(String ver) {
this.hadoopVersion = ver;
}
@@ -221,6 +223,14 @@ public class QTestGenTask extends Task {
this.cleanupScript = cleanupScript;
}
+ public String getUseHBaseMetastore() {
+ return useHBaseMetastore;
+ }
+
+ public void setUseHBaseMetastore(String useHBaseMetastore) {
+ this.useHBaseMetastore = useHBaseMetastore;
+ }
+
public void setHiveRootDirectory(File hiveRootDirectory) {
try {
this.hiveRootDirectory = hiveRootDirectory.getCanonicalPath();
@@ -530,6 +540,7 @@ public class QTestGenTask extends Task {
ctx.put("hadoopVersion", hadoopVersion);
ctx.put("initScript", initScript);
ctx.put("cleanupScript", cleanupScript);
+ ctx.put("useHBaseMetastore", useHBaseMetastore);
File outFile = new File(outDir, className + ".java");
FileWriter writer = new FileWriter(outFile);
http://git-wip-us.apache.org/repos/asf/hive/blob/f014f0da/data/conf/tez/hive-site.xml
----------------------------------------------------------------------
diff --git a/data/conf/tez/hive-site.xml b/data/conf/tez/hive-site.xml
index e0238aa..bcda3ea 100644
--- a/data/conf/tez/hive-site.xml
+++ b/data/conf/tez/hive-site.xml
@@ -253,5 +253,14 @@
</description>
</property>
+<property>
+ <name>hive.metastore.fastpath</name>
+ <value>true</value>
+</property>
+
+<property>
+ <name>hive.metastore.rawstore.impl</name>
+ <value>org.apache.hadoop.hive.metastore.hbase.HBaseStore</value>
+</property>
</configuration>
http://git-wip-us.apache.org/repos/asf/hive/blob/f014f0da/itests/qtest/pom.xml
----------------------------------------------------------------------
diff --git a/itests/qtest/pom.xml b/itests/qtest/pom.xml
index c2cb2f6..664068b 100644
--- a/itests/qtest/pom.xml
+++ b/itests/qtest/pom.xml
@@ -509,7 +509,8 @@
logDirectory="${project.build.directory}/qfile-results/clientpositive/"
hadoopVersion="${active.hadoop.version}"
initScript="${initScript}"
- cleanupScript="q_test_cleanup.sql"/>
+ cleanupScript="q_test_cleanup.sql"
+ useHBaseMetastore="true"/>
<qtestgen hiveRootDirectory="${basedir}/${hive.path.to.root}/"
outputDirectory="${project.build.directory}/generated-test-sources/java/org/apache/hadoop/hive/cli/"
http://git-wip-us.apache.org/repos/asf/hive/blob/f014f0da/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index 358fdbd..9f112ad 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -84,7 +84,6 @@ import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.api.Index;
import org.apache.hadoop.hive.metastore.hbase.HBaseReadWrite;
-import org.apache.hadoop.hive.metastore.hbase.TephraHBaseConnection;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.Utilities;
@@ -109,6 +108,8 @@ import org.apache.hadoop.hive.shims.HadoopShims;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.util.Shell;
import org.apache.hive.common.util.StreamPrinter;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
import org.apache.tools.ant.BuildException;
import org.apache.zookeeper.WatchedEvent;
import org.apache.zookeeper.Watcher;
@@ -166,12 +167,12 @@ public class QTestUtil {
private final String initScript;
private final String cleanupScript;
+ private boolean useHBaseMetastore = false;
public interface SuiteAddTestFunctor {
public void addTestToSuite(TestSuite suite, Object setup, String tName);
}
private HBaseTestingUtility utility;
- private boolean snapshotTaken = false;
static {
for (String srcTable : System.getProperty("test.src.tables", "").trim().split(",")) {
@@ -348,61 +349,46 @@ public class QTestUtil {
return "jceks://file" + new Path(keyDir, "test.jks").toUri();
}
- private void rebuildHBase() throws Exception {
- HBaseAdmin admin = utility.getHBaseAdmin();
- if (!snapshotTaken) {
- for (String tableName : HBaseReadWrite.tableNames) {
- List<byte[]> families = HBaseReadWrite.columnFamilies.get(tableName);
- HTableDescriptor desc = new HTableDescriptor(
- TableName.valueOf(tableName));
- for (byte[] family : families) {
- HColumnDescriptor columnDesc = new HColumnDescriptor(family);
- desc.addFamily(columnDesc);
- }
- try {
- admin.disableTable(tableName);
- admin.deleteTable(tableName);
- } catch (IOException e) {
- System.out.println(e.getMessage());
- }
- admin.createTable(desc);
- }
- } else {
- for (String tableName : HBaseReadWrite.tableNames) {
- admin.disableTable(tableName);
- admin.restoreSnapshot("snapshot_" + tableName);
- admin.enableTable(tableName);
- }
- try {
- db.createDatabase(new org.apache.hadoop.hive.metastore.api.Database(
- DEFAULT_DATABASE_NAME, DEFAULT_DATABASE_COMMENT, new Warehouse(conf)
- .getDefaultDatabasePath(DEFAULT_DATABASE_NAME).toString(), null));
- } catch (Exception e) {
- // Ignore if default database already exist
- }
- SessionState.get().setCurrentDatabase(DEFAULT_DATABASE_NAME);
- }
- admin.close();
- }
-
private void startMiniHBaseCluster() throws Exception {
utility = new HBaseTestingUtility();
utility.startMiniCluster();
conf = new HiveConf(utility.getConfiguration(), Driver.class);
- rebuildHBase();
+ conf = new HiveConf(utility.getConfiguration(), Driver.class);
+ HBaseAdmin admin = utility.getHBaseAdmin();
+ for (String tableName : HBaseReadWrite.tableNames) {
+ List<byte[]> families = HBaseReadWrite.columnFamilies.get(tableName);
+ HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
+ for (byte[] family : families) {
+ HColumnDescriptor columnDesc = new HColumnDescriptor(family);
+ desc.addFamily(columnDesc);
+ }
+ admin.createTable(desc);
+ }
+ admin.close();
HBaseReadWrite.getInstance(conf);
}
public QTestUtil(String outDir, String logDir, MiniClusterType clusterType,
String confDir, String hadoopVer, String initScript, String cleanupScript)
throws Exception {
+ this(outDir, logDir, clusterType, confDir, hadoopVer, initScript, cleanupScript, false);
+ }
+ public QTestUtil(String outDir, String logDir, MiniClusterType clusterType,
+ String confDir, String hadoopVer, String initScript, String cleanupScript, boolean useHBaseMetastore)
+ throws Exception {
this.outDir = outDir;
this.logDir = logDir;
+ this.useHBaseMetastore = useHBaseMetastore;
+
+ Logger hadoopLog = Logger.getLogger("org.apache.hadoop");
+ hadoopLog.setLevel(Level.INFO);
if (confDir != null && !confDir.isEmpty()) {
HiveConf.setHiveSiteLocation(new URL("file://"+ new File(confDir).toURI().getPath() + "/hive-site.xml"));
System.out.println("Setting hive-site: "+HiveConf.getHiveSiteLocation());
}
- startMiniHBaseCluster();
+ if (useHBaseMetastore) {
+ startMiniHBaseCluster();
+ }
conf = new HiveConf(Driver.class);
this.hadoopVer = getHadoopMainVersion(hadoopVer);
qMap = new TreeMap<String, String>();
@@ -491,7 +477,9 @@ public class QTestUtil {
sparkSession = null;
}
}
- utility.shutdownMiniCluster();
+ if (useHBaseMetastore) {
+ utility.shutdownMiniCluster();
+ }
if (mr != null) {
mr.shutdown();
mr = null;
@@ -779,8 +767,6 @@ public class QTestUtil {
return;
}
- rebuildHBase();
-
clearTablesCreatedDuringTests();
clearKeysCreatedInTests();
@@ -880,12 +866,6 @@ public class QTestUtil {
cliDriver.processLine(initCommands);
conf.setBoolean("hive.test.init.phase", false);
-
- HBaseAdmin admin = utility.getHBaseAdmin();
- for (String tableName : HBaseReadWrite.tableNames) {
- admin.snapshot("snapshot_" + tableName, tableName);
- }
- snapshotTaken = true;
}
public void init() throws Exception {
http://git-wip-us.apache.org/repos/asf/hive/blob/f014f0da/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
index f402f73..d9382ff 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
@@ -668,17 +668,19 @@ public class HiveAlterHandler implements AlterHandler {
}
List<ColumnStatisticsObj> statsObjs = cs.getStatsObj();
- for (ColumnStatisticsObj statsObj : statsObjs) {
- boolean found = false;
- for (FieldSchema newCol : newCols) {
- if (statsObj.getColName().equalsIgnoreCase(newCol.getName())
- && statsObj.getColType().equals(newCol.getType())) {
- found = true;
- break;
+ if (statsObjs != null) {
+ for (ColumnStatisticsObj statsObj : statsObjs) {
+ boolean found = false;
+ for (FieldSchema newCol : newCols) {
+ if (statsObj.getColName().equalsIgnoreCase(newCol.getName())
+ && statsObj.getColType().equals(newCol.getType())) {
+ found = true;
+ break;
+ }
+ }
+ if (!found) {
+ msdb.deleteTableColumnStatistics(dbName, tableName, statsObj.getColName());
}
- }
- if (!found) {
- msdb.deleteTableColumnStatistics(dbName, tableName, statsObj.getColName());
}
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/f014f0da/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
index f1336dc..8a1448c 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
@@ -680,7 +680,13 @@ public class HBaseReadWrite {
firstStar = i;
break;
} else {
- keyElements.add(partVals.get(i));
+ // empty string equals to null partition,
+ // means star
+ if (partVals.get(i).equals("")) {
+ break;
+ } else {
+ keyElements.add(partVals.get(i));
+ }
}
}
@@ -693,7 +699,7 @@ public class HBaseReadWrite {
}
keyPrefix = HBaseUtils.buildPartitionKey(dbName, tableName,
HBaseUtils.getPartitionKeyTypes(table.getPartitionKeys().subList(0, keyElements.size()-2)),
- keyElements.subList(0, keyElements.size()-2));
+ keyElements.subList(2, keyElements.size()));
// Now, build a filter out of the remaining keys
List<PartitionKeyComparator.Range> ranges = new ArrayList<PartitionKeyComparator.Range>();
@@ -809,7 +815,7 @@ public class HBaseReadWrite {
for (int i = 0; i < numToFetch && iter.hasNext(); i++) {
Result result = iter.next();
HBaseUtils.StorageDescriptorParts sdParts = HBaseUtils.deserializePartition(dbName, tableName,
- tablePartitions, result.getRow(), result.getValue(CATALOG_CF, CATALOG_COL));
+ tablePartitions, result.getRow(), result.getValue(CATALOG_CF, CATALOG_COL), staticConf);
StorageDescriptor sd = getStorageDescriptor(sdParts.sdHash);
HBaseUtils.assembleStorageDescriptor(sd, sdParts);
parts.add(sdParts.containingPartition);
@@ -1604,9 +1610,10 @@ public class HBaseReadWrite {
// recontruct the key. We have to pull the dbName and tableName out of the key to
// find the partition values.
byte[] key = results[i].getRow();
- String[] reconstructedKey = HBaseUtils.parseKey(key);
+ List<String> reconstructedKey = HBaseUtils.parseKey(key, HBaseUtils.getPartitionNames(getTable(dbName, tblName).getPartitionKeys()),
+ HBaseUtils.getPartitionKeyTypes(getTable(dbName, tblName).getPartitionKeys()));
List<String> reconstructedPartVals =
- Arrays.asList(reconstructedKey).subList(2, reconstructedKey.length);
+ reconstructedKey.subList(2, reconstructedKey.size());
String partName = valToPartMap.get(reconstructedPartVals);
assert partName != null;
csd.setIsTblLevel(false);
http://git-wip-us.apache.org/repos/asf/hive/blob/f014f0da/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
index f30fcab..568a347 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
@@ -20,15 +20,18 @@ package org.apache.hadoop.hive.metastore.hbase;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.cache.CacheLoader;
+
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.common.FileUtils;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStore;
import org.apache.hadoop.hive.metastore.PartFilterExprUtil;
import org.apache.hadoop.hive.metastore.PartitionExpressionProxy;
import org.apache.hadoop.hive.metastore.RawStore;
+import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.api.AggrStats;
import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId;
@@ -64,6 +67,9 @@ import org.apache.hadoop.hive.metastore.hbase.HBaseFilterPlanUtil.PlanResult;
import org.apache.hadoop.hive.metastore.hbase.HBaseFilterPlanUtil.ScanPlan;
import org.apache.hadoop.hive.metastore.parser.ExpressionTree;
import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hive.common.util.HiveStringUtils;
import org.apache.thrift.TException;
import java.io.IOException;
@@ -71,6 +77,7 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -150,7 +157,7 @@ public class HBaseStore implements RawStore {
boolean commit = false;
openTransaction();
try {
- Database db = getHBase().getDb(name);
+ Database db = getHBase().getDb(HiveStringUtils.normalizeIdentifier(name));
if (db == null) {
throw new NoSuchObjectException("Unable to find db " + name);
}
@@ -169,7 +176,7 @@ public class HBaseStore implements RawStore {
boolean commit = false;
openTransaction();
try {
- getHBase().deleteDb(dbname);
+ getHBase().deleteDb(HiveStringUtils.normalizeIdentifier(dbname));
commit = true;
return true;
} catch (IOException e) {
@@ -259,7 +266,8 @@ public class HBaseStore implements RawStore {
boolean commit = false;
openTransaction();
try {
- getHBase().deleteTable(dbName, tableName);
+ getHBase().deleteTable(HiveStringUtils.normalizeIdentifier(dbName),
+ HiveStringUtils.normalizeIdentifier(tableName));
commit = true;
return true;
} catch (IOException e) {
@@ -275,7 +283,8 @@ public class HBaseStore implements RawStore {
boolean commit = false;
openTransaction();
try {
- Table table = getHBase().getTable(dbName, tableName);
+ Table table = getHBase().getTable(HiveStringUtils.normalizeIdentifier(dbName),
+ HiveStringUtils.normalizeIdentifier(tableName));
if (table == null) {
LOG.debug("Unable to find table " + tableNameForErrorMsg(dbName, tableName));
}
@@ -334,7 +343,8 @@ public class HBaseStore implements RawStore {
boolean commit = false;
openTransaction();
try {
- Partition part = getHBase().getPartition(dbName, tableName, part_vals);
+ Partition part = getHBase().getPartition(HiveStringUtils.normalizeIdentifier(dbName),
+ HiveStringUtils.normalizeIdentifier(tableName), part_vals);
if (part == null) {
throw new NoSuchObjectException("Unable to find partition " +
partNameForErrorMsg(dbName, tableName, part_vals));
@@ -355,7 +365,8 @@ public class HBaseStore implements RawStore {
boolean commit = false;
openTransaction();
try {
- boolean exists = getHBase().getPartition(dbName, tableName, part_vals) != null;
+ boolean exists = getHBase().getPartition(HiveStringUtils.normalizeIdentifier(dbName),
+ HiveStringUtils.normalizeIdentifier(tableName), part_vals) != null;
commit = true;
return exists;
} catch (IOException e) {
@@ -372,9 +383,11 @@ public class HBaseStore implements RawStore {
boolean commit = false;
openTransaction();
try {
- getHBase().deletePartition(dbName, tableName, part_vals);
+ getHBase().deletePartition(HiveStringUtils.normalizeIdentifier(dbName),
+ HiveStringUtils.normalizeIdentifier(tableName), part_vals);
// Drop any cached stats that reference this partitions
- getHBase().getStatsCache().invalidate(dbName, tableName,
+ getHBase().getStatsCache().invalidate(HiveStringUtils.normalizeIdentifier(dbName),
+ HiveStringUtils.normalizeIdentifier(tableName),
buildExternalPartName(dbName, tableName, part_vals));
commit = true;
return true;
@@ -393,7 +406,8 @@ public class HBaseStore implements RawStore {
boolean commit = false;
openTransaction();
try {
- List<Partition> parts = getHBase().scanPartitionsInTable(dbName, tableName, max);
+ List<Partition> parts = getHBase().scanPartitionsInTable(HiveStringUtils.normalizeIdentifier(dbName),
+ HiveStringUtils.normalizeIdentifier(tableName), max);
commit = true;
return parts;
} catch (IOException e) {
@@ -410,7 +424,8 @@ public class HBaseStore implements RawStore {
boolean commit = false;
openTransaction();
try {
- getHBase().replaceTable(getHBase().getTable(dbname, name), newTable);
+ getHBase().replaceTable(getHBase().getTable(HiveStringUtils.normalizeIdentifier(dbname),
+ HiveStringUtils.normalizeIdentifier(name)), newTable);
if (newTable.getPartitionKeys() != null && newTable.getPartitionKeys().size() > 0
&& !name.equals(newTable.getTableName())) {
// They renamed the table, so we need to change each partition as well, since it changes
@@ -443,7 +458,8 @@ public class HBaseStore implements RawStore {
boolean commit = false;
openTransaction();
try {
- List<Table> tables = getHBase().scanTables(dbName, likeToRegex(pattern));
+ List<Table> tables = getHBase().scanTables(HiveStringUtils.normalizeIdentifier(dbName),
+ likeToRegex(pattern));
List<String> tableNames = new ArrayList<String>(tables.size());
for (Table table : tables) tableNames.add(table.getTableName());
commit = true;
@@ -462,7 +478,12 @@ public class HBaseStore implements RawStore {
boolean commit = false;
openTransaction();
try {
- List<Table> tables = getHBase().getTables(dbname, tableNames);
+ List<String> normalizedTableNames = new ArrayList<String>(tableNames.size());
+ for (String tableName : tableNames) {
+ normalizedTableNames.add(HiveStringUtils.normalizeIdentifier(tableName));
+ }
+ List<Table> tables = getHBase().getTables(HiveStringUtils.normalizeIdentifier(dbname),
+ normalizedTableNames);
commit = true;
return tables;
} catch (IOException e) {
@@ -491,10 +512,12 @@ public class HBaseStore implements RawStore {
boolean commit = false;
openTransaction();
try {
- List<Partition> parts = getHBase().scanPartitionsInTable(db_name, tbl_name, max_parts);
+ List<Partition> parts = getHBase().scanPartitionsInTable(HiveStringUtils.normalizeIdentifier(db_name),
+ HiveStringUtils.normalizeIdentifier(tbl_name), max_parts);
if (parts == null) return null;
List<String> names = new ArrayList<String>(parts.size());
- Table table = getHBase().getTable(db_name, tbl_name);
+ Table table = getHBase().getTable(HiveStringUtils.normalizeIdentifier(db_name),
+ HiveStringUtils.normalizeIdentifier(tbl_name));
for (Partition p : parts) {
names.add(buildExternalPartName(table, p));
}
@@ -521,10 +544,12 @@ public class HBaseStore implements RawStore {
boolean commit = false;
openTransaction();
try {
- Partition oldPart = getHBase().getPartition(db_name, tbl_name, part_vals);
+ Partition oldPart = getHBase().getPartition(HiveStringUtils.normalizeIdentifier(db_name),
+ HiveStringUtils.normalizeIdentifier(tbl_name), part_vals);
getHBase().replacePartition(oldPart, new_part);
// Drop any cached stats that reference this partitions
- getHBase().getStatsCache().invalidate(db_name, tbl_name,
+ getHBase().getStatsCache().invalidate(HiveStringUtils.normalizeIdentifier(db_name),
+ HiveStringUtils.normalizeIdentifier(tbl_name),
buildExternalPartName(db_name, tbl_name, part_vals));
commit = true;
} catch (IOException e) {
@@ -542,11 +567,14 @@ public class HBaseStore implements RawStore {
boolean commit = false;
openTransaction();
try {
- List<Partition> oldParts = getHBase().getPartitions(db_name, tbl_name,
- HBaseUtils.getPartitionKeyTypes(getTable(db_name, tbl_name).getPartitionKeys()), part_vals_list);
+ List<Partition> oldParts = getHBase().getPartitions(HiveStringUtils.normalizeIdentifier(db_name),
+ HiveStringUtils.normalizeIdentifier(tbl_name),
+ HBaseUtils.getPartitionKeyTypes(getTable(HiveStringUtils.normalizeIdentifier(db_name),
+ HiveStringUtils.normalizeIdentifier(tbl_name)).getPartitionKeys()), part_vals_list);
getHBase().replacePartitions(oldParts, new_parts);
for (List<String> part_vals : part_vals_list) {
- getHBase().getStatsCache().invalidate(db_name, tbl_name,
+ getHBase().getStatsCache().invalidate(HiveStringUtils.normalizeIdentifier(db_name),
+ HiveStringUtils.normalizeIdentifier(tbl_name),
buildExternalPartName(db_name, tbl_name, part_vals));
}
commit = true;
@@ -604,7 +632,8 @@ public class HBaseStore implements RawStore {
boolean commit = false;
openTransaction();
try {
- getPartitionsByExprInternal(dbName, tblName, exprTree, maxParts, result);
+ getPartitionsByExprInternal(HiveStringUtils.normalizeIdentifier(dbName),
+ HiveStringUtils.normalizeIdentifier(tblName), exprTree, maxParts, result);
return result;
} finally {
commitOrRoleBack(commit);
@@ -616,22 +645,62 @@ public class HBaseStore implements RawStore {
String defaultPartitionName, short maxParts,
List<Partition> result) throws TException {
final ExpressionTree exprTree = PartFilterExprUtil.makeExpressionTree(expressionProxy, expr);
- // TODO: investigate if there should be any role for defaultPartitionName in this
- // implementation. direct sql code path in ObjectStore does not use it.
-
+ dbName = HiveStringUtils.normalizeIdentifier(dbName);
+ tblName = HiveStringUtils.normalizeIdentifier(tblName);
+ Table table = getTable(dbName, tblName);
boolean commit = false;
openTransaction();
try {
- return getPartitionsByExprInternal(dbName, tblName, exprTree, maxParts, result);
+ if (exprTree == null) {
+ List<String> partNames = new LinkedList<String>();
+ boolean hasUnknownPartitions = getPartitionNamesPrunedByExprNoTxn(
+ table, expr, defaultPartitionName, maxParts, partNames);
+ result.addAll(getPartitionsByNames(dbName, tblName, partNames));
+ return hasUnknownPartitions;
+ } else {
+ return getPartitionsByExprInternal(dbName, tblName, exprTree, maxParts, result);
+ }
} finally {
commitOrRoleBack(commit);
}
}
+ /**
+ * Gets the partition names from a table, pruned using an expression.
+ * @param table Table.
+ * @param expr Expression.
+ * @param defaultPartName Default partition name from job config, if any.
+ * @param maxParts Maximum number of partition names to return.
+ * @param result The resulting names.
+ * @return Whether the result contains any unknown partitions.
+ * @throws NoSuchObjectException
+ */
+ private boolean getPartitionNamesPrunedByExprNoTxn(Table table, byte[] expr,
+ String defaultPartName, short maxParts, List<String> result) throws MetaException, NoSuchObjectException {
+ List<Partition> parts = getPartitions(
+ table.getDbName(), table.getTableName(), maxParts);
+ for (Partition part : parts) {
+ result.add(Warehouse.makePartName(table.getPartitionKeys(), part.getValues()));
+ }
+ List<String> columnNames = new ArrayList<String>();
+ List<PrimitiveTypeInfo> typeInfos = new ArrayList<PrimitiveTypeInfo>();
+ for (FieldSchema fs : table.getPartitionKeys()) {
+ columnNames.add(fs.getName());
+ typeInfos.add(TypeInfoFactory.getPrimitiveTypeInfo(fs.getType()));
+ }
+ if (defaultPartName == null || defaultPartName.isEmpty()) {
+ defaultPartName = HiveConf.getVar(getConf(), HiveConf.ConfVars.DEFAULTPARTITIONNAME);
+ }
+ return expressionProxy.filterPartitionsByExpr(
+ columnNames, typeInfos, expr, defaultPartName, result);
+ }
+
private boolean getPartitionsByExprInternal(String dbName, String tblName,
ExpressionTree exprTree, short maxParts, List<Partition> result) throws MetaException,
NoSuchObjectException {
+ dbName = HiveStringUtils.normalizeIdentifier(dbName);
+ tblName = HiveStringUtils.normalizeIdentifier(tblName);
Table table = getTable(dbName, tblName);
if (table == null) {
throw new NoSuchObjectException("Unable to find table " + dbName + "." + tblName);
@@ -1453,7 +1522,8 @@ public class HBaseStore implements RawStore {
listPartitionsPsWithAuth(db_name, tbl_name, part_vals, max_parts, null, null);
List<String> partNames = new ArrayList<String>(parts.size());
for (Partition part : parts) {
- partNames.add(buildExternalPartName(db_name, tbl_name, part.getValues()));
+ partNames.add(buildExternalPartName(HiveStringUtils.normalizeIdentifier(db_name),
+ HiveStringUtils.normalizeIdentifier(tbl_name), part.getValues()));
}
return partNames;
}
@@ -1468,7 +1538,8 @@ public class HBaseStore implements RawStore {
boolean commit = false;
openTransaction();
try {
- List<Partition> parts = getHBase().scanPartitions(db_name, tbl_name, part_vals, max_parts);
+ List<Partition> parts = getHBase().scanPartitions(HiveStringUtils.normalizeIdentifier(db_name),
+ HiveStringUtils.normalizeIdentifier(tbl_name), part_vals, max_parts);
commit = true;
return parts;
} catch (IOException e) {
@@ -1596,7 +1667,7 @@ public class HBaseStore implements RawStore {
getHBase().getStatsCache().get(dbName, tblName, partNames, colName);
if (oneCol.getColStatsSize() > 0) {
assert oneCol.getColStatsSize() == 1;
- aggrStats.setPartsFound(aggrStats.getPartsFound() + oneCol.getPartsFound());
+ aggrStats.setPartsFound(oneCol.getPartsFound());
aggrStats.addToColStats(oneCol.getColStats().get(0));
}
} catch (CacheLoader.InvalidCacheLoadException e) {
@@ -2204,7 +2275,7 @@ public class HBaseStore implements RawStore {
List<String> vals = new ArrayList<String>();
String[] kvp = name.split("/");
for (String kv : kvp) {
- vals.add(kv.substring(kv.indexOf('=') + 1));
+ vals.add(FileUtils.unescapePathName(kv.substring(kv.indexOf('=') + 1)));
}
return vals;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/f014f0da/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseUtils.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseUtils.java
index 841afd4..cc90a76 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseUtils.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseUtils.java
@@ -26,6 +26,7 @@ import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.AggrStats;
import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData;
import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
@@ -67,6 +68,7 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hive.common.util.BloomFilter;
+import org.apache.hive.common.util.HiveStringUtils;
import java.io.IOException;
import java.nio.charset.Charset;
@@ -115,9 +117,22 @@ class HBaseUtils {
return protoKey.getBytes(ENCODING);
}
- static String[] parseKey(byte[] serialized) {
- String munged = new String(serialized, ENCODING);
- return munged.split(KEY_SEPARATOR_STR);
+ static List<String> parseKey(byte[] serialized, List<String> partNames, List<String> partTypes) {
+ BinarySortableSerDe serDe = new BinarySortableSerDe();
+ Properties props = new Properties();
+ props.setProperty(serdeConstants.LIST_COLUMNS, "dbName,tableName," + StringUtils.join(partNames, ","));
+ props.setProperty(serdeConstants.LIST_COLUMN_TYPES, "string,string," + StringUtils.join(partTypes, ","));
+ List<String> partVals = null;
+ try {
+ serDe.initialize(new Configuration(), props);
+ List deserializedkeys = ((List)serDe.deserialize(new BytesWritable(serialized)));
+ partVals = new ArrayList<String>();
+ for (Object deserializedkey : deserializedkeys) {
+ partVals.add(deserializedkey.toString());
+ }
+ } catch (SerDeException e) {
+ }
+ return partVals;
}
private static HbaseMetastoreProto.Parameters buildParameters(Map<String, String> params) {
@@ -240,9 +255,16 @@ class HBaseUtils {
private static PrincipalPrivilegeSet buildPrincipalPrivilegeSet(
HbaseMetastoreProto.PrincipalPrivilegeSet proto) throws InvalidProtocolBufferException {
- PrincipalPrivilegeSet pps = new PrincipalPrivilegeSet();
- pps.setUserPrivileges(convertPrincipalPrivilegeSetEntries(proto.getUsersList()));
- pps.setRolePrivileges(convertPrincipalPrivilegeSetEntries(proto.getRolesList()));
+ PrincipalPrivilegeSet pps = null;
+ if (!proto.getUsersList().isEmpty() || !proto.getRolesList().isEmpty()) {
+ pps = new PrincipalPrivilegeSet();
+ if (!proto.getUsersList().isEmpty()) {
+ pps.setUserPrivileges(convertPrincipalPrivilegeSetEntries(proto.getUsersList()));
+ }
+ if (!proto.getRolesList().isEmpty()) {
+ pps.setRolePrivileges(convertPrincipalPrivilegeSetEntries(proto.getRolesList()));
+ }
+ }
return pps;
}
/**
@@ -339,7 +361,7 @@ class HBaseUtils {
*/
static byte[][] serializeDatabase(Database db) {
byte[][] result = new byte[2][];
- result[0] = buildKey(db.getName());
+ result[0] = buildKey(HiveStringUtils.normalizeIdentifier(db.getName()));
HbaseMetastoreProto.Database.Builder builder = HbaseMetastoreProto.Database.newBuilder();
if (db.getDescription() != null) builder.setDescription(db.getDescription());
@@ -696,8 +718,10 @@ class HBaseUtils {
sd.setNumBuckets(proto.getNumBuckets());
if (proto.hasSerdeInfo()) {
SerDeInfo serde = new SerDeInfo();
- serde.setName(proto.getSerdeInfo().getName());
- serde.setSerializationLib(proto.getSerdeInfo().getSerializationLib());
+ serde.setName(proto.getSerdeInfo().hasName()?
+ proto.getSerdeInfo().getName():null);
+ serde.setSerializationLib(proto.getSerdeInfo().hasSerializationLib()?
+ proto.getSerdeInfo().getSerializationLib():null);
serde.setParameters(buildParameters(proto.getSerdeInfo().getParameters()));
sd.setSerdeInfo(serde);
}
@@ -848,8 +872,8 @@ class HBaseUtils {
* @return A struct that contains the partition plus parts of the storage descriptor
*/
static StorageDescriptorParts deserializePartition(String dbName, String tableName, List<FieldSchema> partitions,
- byte[] key, byte[] serialized) throws InvalidProtocolBufferException {
- List keys = deserializePartitionKey(partitions, key);
+ byte[] key, byte[] serialized, Configuration conf) throws InvalidProtocolBufferException {
+ List keys = deserializePartitionKey(partitions, key, conf);
return deserializePartition(dbName, tableName, keys, serialized);
}
@@ -886,7 +910,8 @@ class HBaseUtils {
return k.split(KEY_SEPARATOR_STR);
}
- private static List<String> deserializePartitionKey(List<FieldSchema> partitions, byte[] key) {
+ private static List<String> deserializePartitionKey(List<FieldSchema> partitions, byte[] key,
+ Configuration conf) {
StringBuffer names = new StringBuffer();
names.append("dbName,tableName,");
StringBuffer types = new StringBuffer();
@@ -908,7 +933,8 @@ class HBaseUtils {
List deserializedkeys = ((List)serDe.deserialize(new BytesWritable(key))).subList(2, partitions.size()+2);
List<String> partitionKeys = new ArrayList<String>();
for (Object deserializedKey : deserializedkeys) {
- partitionKeys.add(deserializedKey.toString());
+ partitionKeys.add(deserializedKey!=null?deserializedKey.toString():
+ HiveConf.getVar(conf, HiveConf.ConfVars.DEFAULTPARTITIONNAME));
}
return partitionKeys;
} catch (SerDeException e) {
@@ -924,7 +950,8 @@ class HBaseUtils {
*/
static byte[][] serializeTable(Table table, byte[] sdHash) {
byte[][] result = new byte[2][];
- result[0] = buildKey(table.getDbName(), table.getTableName());
+ result[0] = buildKey(HiveStringUtils.normalizeIdentifier(table.getDbName()),
+ HiveStringUtils.normalizeIdentifier(table.getTableName()));
HbaseMetastoreProto.Table.Builder builder = HbaseMetastoreProto.Table.newBuilder();
if (table.getOwner() != null) builder.setOwner(table.getOwner());
builder
@@ -952,7 +979,10 @@ class HBaseUtils {
if (table.getPrivileges() != null) {
builder.setPrivileges(buildPrincipalPrivilegeSet(table.getPrivileges()));
}
- builder.setIsTemporary(table.isTemporary());
+ // Set only if table is temporary
+ if (table.isTemporary()) {
+ builder.setIsTemporary(table.isTemporary());
+ }
result[1] = builder.build().toByteArray();
return result;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/f014f0da/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/StatsCache.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/StatsCache.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/StatsCache.java
index 0d3ed40..42efe94 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/StatsCache.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/StatsCache.java
@@ -99,7 +99,7 @@ class StatsCache {
for (ColumnStatisticsObj cso : cs.getStatsObj()) {
if (statsObj == null) {
statsObj = ColumnStatsAggregatorFactory.newColumnStaticsObj(key.colName,
- cso.getStatsData().getSetField());
+ cso.getColType(), cso.getStatsData().getSetField());
}
if (aggregator == null) {
aggregator = ColumnStatsAggregatorFactory.getColumnStatsAggregator(
http://git-wip-us.apache.org/repos/asf/hive/blob/f014f0da/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/ColumnStatsAggregatorFactory.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/ColumnStatsAggregatorFactory.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/ColumnStatsAggregatorFactory.java
index ebecfe3..a8dbc1f 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/ColumnStatsAggregatorFactory.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/ColumnStatsAggregatorFactory.java
@@ -53,39 +53,34 @@ public class ColumnStatsAggregatorFactory {
}
}
- public static ColumnStatisticsObj newColumnStaticsObj(String colName, _Fields type) {
+ public static ColumnStatisticsObj newColumnStaticsObj(String colName, String colType, _Fields type) {
ColumnStatisticsObj cso = new ColumnStatisticsObj();
ColumnStatisticsData csd = new ColumnStatisticsData();
cso.setColName(colName);
+ cso.setColType(colType);
switch (type) {
case BOOLEAN_STATS:
csd.setBooleanStats(new BooleanColumnStatsData());
- cso.setColType("boolean");
break;
case LONG_STATS:
csd.setLongStats(new LongColumnStatsData());
- cso.setColType("long");
break;
case DOUBLE_STATS:
csd.setDoubleStats(new DoubleColumnStatsData());
- cso.setColType("double");
break;
case STRING_STATS:
csd.setStringStats(new StringColumnStatsData());
- cso.setColType("string");
break;
case BINARY_STATS:
csd.setBinaryStats(new BinaryColumnStatsData());
- cso.setColType("binary");
break;
case DECIMAL_STATS:
csd.setDecimalStats(new DecimalColumnStatsData());
- cso.setColType("decimal");
break;
default:
http://git-wip-us.apache.org/repos/asf/hive/blob/f014f0da/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
index 55aea0e..351cb2b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
@@ -737,6 +737,9 @@ public class StatsUtils {
}
private static List<ColStatistics> convertColStats(List<ColumnStatisticsObj> colStats, String tabName) {
+ if (colStats==null) {
+ return new ArrayList<ColStatistics>();
+ }
List<ColStatistics> stats = new ArrayList<ColStatistics>(colStats.size());
for (ColumnStatisticsObj statObj : colStats) {
ColStatistics cs = getColStatistics(statObj, tabName, statObj.getColName());
http://git-wip-us.apache.org/repos/asf/hive/blob/f014f0da/ql/src/test/queries/clientpositive/dynpart_sort_opt_vectorization.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/dynpart_sort_opt_vectorization.q b/ql/src/test/queries/clientpositive/dynpart_sort_opt_vectorization.q
index 8001081..7e94f23 100644
--- a/ql/src/test/queries/clientpositive/dynpart_sort_opt_vectorization.q
+++ b/ql/src/test/queries/clientpositive/dynpart_sort_opt_vectorization.q
@@ -123,6 +123,7 @@ insert overwrite table over1k_part2_orc partition(ds="foo",t) select si,i,b,f,t
desc formatted over1k_part2_orc partition(ds="foo",t=27);
desc formatted over1k_part2_orc partition(ds="foo",t="__HIVE_DEFAULT_PARTITION__");
+-- SORT_BEFORE_DIFF
select * from over1k_part2_orc;
select count(*) from over1k_part2_orc;
@@ -132,6 +133,7 @@ insert overwrite table over1k_part2_orc partition(ds="foo",t) select si,i,b,f,t
desc formatted over1k_part2_orc partition(ds="foo",t=27);
desc formatted over1k_part2_orc partition(ds="foo",t="__HIVE_DEFAULT_PARTITION__");
+-- SORT_BEFORE_DIFF
select * from over1k_part2_orc;
select count(*) from over1k_part2_orc;
http://git-wip-us.apache.org/repos/asf/hive/blob/f014f0da/ql/src/test/queries/clientpositive/dynpart_sort_optimization.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/dynpart_sort_optimization.q b/ql/src/test/queries/clientpositive/dynpart_sort_optimization.q
index f842efe..ea670e9 100644
--- a/ql/src/test/queries/clientpositive/dynpart_sort_optimization.q
+++ b/ql/src/test/queries/clientpositive/dynpart_sort_optimization.q
@@ -117,6 +117,7 @@ insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from
desc formatted over1k_part2 partition(ds="foo",t=27);
desc formatted over1k_part2 partition(ds="foo",t="__HIVE_DEFAULT_PARTITION__");
+-- SORT_BEFORE_DIFF
select * from over1k_part2;
select count(*) from over1k_part2;
@@ -126,6 +127,7 @@ insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from
desc formatted over1k_part2 partition(ds="foo",t=27);
desc formatted over1k_part2 partition(ds="foo",t="__HIVE_DEFAULT_PARTITION__");
+-- SORT_BEFORE_DIFF
select * from over1k_part2;
select count(*) from over1k_part2;
http://git-wip-us.apache.org/repos/asf/hive/blob/f014f0da/ql/src/test/results/clientpositive/tez/dynpart_sort_opt_vectorization.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/dynpart_sort_opt_vectorization.q.out b/ql/src/test/results/clientpositive/tez/dynpart_sort_opt_vectorization.q.out
index 4451046..f0fc221 100644
--- a/ql/src/test/results/clientpositive/tez/dynpart_sort_opt_vectorization.q.out
+++ b/ql/src/test/results/clientpositive/tez/dynpart_sort_opt_vectorization.q.out
@@ -1835,13 +1835,15 @@ Bucket Columns: []
Sort Columns: []
Storage Desc Params:
serialization.format 1
-PREHOOK: query: select * from over1k_part2_orc
+PREHOOK: query: -- SORT_BEFORE_DIFF
+select * from over1k_part2_orc
PREHOOK: type: QUERY
PREHOOK: Input: default@over1k_part2_orc
PREHOOK: Input: default@over1k_part2_orc@ds=foo/t=27
PREHOOK: Input: default@over1k_part2_orc@ds=foo/t=__HIVE_DEFAULT_PARTITION__
#### A masked pattern was here ####
-POSTHOOK: query: select * from over1k_part2_orc
+POSTHOOK: query: -- SORT_BEFORE_DIFF
+select * from over1k_part2_orc
POSTHOOK: type: QUERY
POSTHOOK: Input: default@over1k_part2_orc
POSTHOOK: Input: default@over1k_part2_orc@ds=foo/t=27
@@ -1980,13 +1982,15 @@ Bucket Columns: []
Sort Columns: []
Storage Desc Params:
serialization.format 1
-PREHOOK: query: select * from over1k_part2_orc
+PREHOOK: query: -- SORT_BEFORE_DIFF
+select * from over1k_part2_orc
PREHOOK: type: QUERY
PREHOOK: Input: default@over1k_part2_orc
PREHOOK: Input: default@over1k_part2_orc@ds=foo/t=27
PREHOOK: Input: default@over1k_part2_orc@ds=foo/t=__HIVE_DEFAULT_PARTITION__
#### A masked pattern was here ####
-POSTHOOK: query: select * from over1k_part2_orc
+POSTHOOK: query: -- SORT_BEFORE_DIFF
+select * from over1k_part2_orc
POSTHOOK: type: QUERY
POSTHOOK: Input: default@over1k_part2_orc
POSTHOOK: Input: default@over1k_part2_orc@ds=foo/t=27
http://git-wip-us.apache.org/repos/asf/hive/blob/f014f0da/ql/src/test/results/clientpositive/tez/dynpart_sort_optimization.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/dynpart_sort_optimization.q.out b/ql/src/test/results/clientpositive/tez/dynpart_sort_optimization.q.out
index cb001b9..8d4c1b7 100644
--- a/ql/src/test/results/clientpositive/tez/dynpart_sort_optimization.q.out
+++ b/ql/src/test/results/clientpositive/tez/dynpart_sort_optimization.q.out
@@ -1735,13 +1735,15 @@ Bucket Columns: []
Sort Columns: []
Storage Desc Params:
serialization.format 1
-PREHOOK: query: select * from over1k_part2
+PREHOOK: query: -- SORT_BEFORE_DIFF
+select * from over1k_part2
PREHOOK: type: QUERY
PREHOOK: Input: default@over1k_part2
PREHOOK: Input: default@over1k_part2@ds=foo/t=27
PREHOOK: Input: default@over1k_part2@ds=foo/t=__HIVE_DEFAULT_PARTITION__
#### A masked pattern was here ####
-POSTHOOK: query: select * from over1k_part2
+POSTHOOK: query: -- SORT_BEFORE_DIFF
+select * from over1k_part2
POSTHOOK: type: QUERY
POSTHOOK: Input: default@over1k_part2
POSTHOOK: Input: default@over1k_part2@ds=foo/t=27
@@ -1880,13 +1882,15 @@ Bucket Columns: []
Sort Columns: []
Storage Desc Params:
serialization.format 1
-PREHOOK: query: select * from over1k_part2
+PREHOOK: query: -- SORT_BEFORE_DIFF
+select * from over1k_part2
PREHOOK: type: QUERY
PREHOOK: Input: default@over1k_part2
PREHOOK: Input: default@over1k_part2@ds=foo/t=27
PREHOOK: Input: default@over1k_part2@ds=foo/t=__HIVE_DEFAULT_PARTITION__
#### A masked pattern was here ####
-POSTHOOK: query: select * from over1k_part2
+POSTHOOK: query: -- SORT_BEFORE_DIFF
+select * from over1k_part2
POSTHOOK: type: QUERY
POSTHOOK: Input: default@over1k_part2
POSTHOOK: Input: default@over1k_part2@ds=foo/t=27
http://git-wip-us.apache.org/repos/asf/hive/blob/f014f0da/ql/src/test/templates/TestCliDriver.vm
----------------------------------------------------------------------
diff --git a/ql/src/test/templates/TestCliDriver.vm b/ql/src/test/templates/TestCliDriver.vm
index ae449c5..01745da 100644
--- a/ql/src/test/templates/TestCliDriver.vm
+++ b/ql/src/test/templates/TestCliDriver.vm
@@ -45,13 +45,14 @@ public class $className extends TestCase {
String hiveConfDir = "$hiveConfDir";
String initScript = "$initScript";
String cleanupScript = "$cleanupScript";
+ boolean useHBaseMetastore = Boolean.valueOf("$useHBaseMetastore");
try {
String hadoopVer = "$hadoopVersion";
if (!hiveConfDir.isEmpty()) {
hiveConfDir = HIVE_ROOT + hiveConfDir;
}
qt = new QTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR,
- hiveConfDir, hadoopVer, initScript, cleanupScript);
+ hiveConfDir, hadoopVer, initScript, cleanupScript, useHBaseMetastore);
// do a one time initialization
qt.cleanUp();