You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/08/16 03:22:02 UTC
svn commit: r1514554 [3/18] - in /hive/branches/vectorization: ./
beeline/src/java/org/apache/hive/beeline/
cli/src/java/org/apache/hadoop/hive/cli/
cli/src/test/org/apache/hadoop/hive/cli/
common/src/java/org/apache/hadoop/hive/conf/ conf/ contrib/src...
Modified: hive/branches/vectorization/conf/hive-default.xml.template
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/conf/hive-default.xml.template?rev=1514554&r1=1514553&r2=1514554&view=diff
==============================================================================
--- hive/branches/vectorization/conf/hive-default.xml.template (original)
+++ hive/branches/vectorization/conf/hive-default.xml.template Fri Aug 16 01:21:54 2013
@@ -447,6 +447,12 @@
</property>
<property>
+ <name>hive.session.history.enabled</name>
+ <value>false</value>
+ <description>Whether to log hive query, query plan, runtime statistics etc </description>
+</property>
+
+<property>
<name>hive.map.aggr.hash.min.reduction</name>
<value>0.5</value>
<description>Hash aggregation will be turned off if the ratio between hash
@@ -860,16 +866,6 @@
</property>
<property>
- <name>hive.optimize.mapjoin.mapreduce</name>
- <value>false</value>
- <description>If hive.auto.convert.join is off, this parameter does not take
- affect. If it is on, and if there are map-join jobs followed by a map-reduce
- job (for e.g a group by), each map-only job is merged with the following
- map-reduce job.
- </description>
-</property>
-
-<property>
<name>hive.script.auto.progress</name>
<value>false</value>
<description>Whether Hive Tranform/Map/Reduce Clause should automatically send progress information to TaskTracker to avoid the task getting killed because of inactivity. Hive sends progress information when the script is outputting to stderr. This option removes the need of periodically producing stderr messages, but users should be cautious because this may prevent infinite loops in the scripts to be killed by TaskTracker. </description>
@@ -1711,6 +1707,15 @@
</property>
<property>
+ <name>hive.exec.orc.dictionary.key.size.threshold</name>
+ <value>0.8</value>
+ <description>
+ If the number of keys in a dictionary is greater than this fraction of the total number of
+ non-null rows, turn off dictionary encoding. Use 1 to always use dictionary encoding.
+ </description>
+</property>
+
+<property>
<name>hive.multi.insert.move.tasks.share.dependencies</name>
<value>false</value>
<description>
@@ -1929,6 +1934,22 @@
</description>
</property>
+<property>
+ <name>hive.server2.table.type.mapping</name>
+ <value>HIVE</value>
+ <description>
+ This setting reflects how HiveServer will report the table types for JDBC and other
+ client implementations that retrieves the available tables and supported table types
+ HIVE : Exposes the hive's native table tyes like MANAGED_TABLE, EXTERNAL_TABLE, VIRTUAL_VIEW
+ CLASSIC : More generic types like TABLE and VIEW
+ </description>
+</property>
+
+<property>
+ <name>hive.server2.thrift.sasl.qop</name>
+ <value>auth</auth>
+ <description>Sasl QOP value; one of 'auth', 'auth-int' and 'auth-conf'</description>
+</property>
</configuration>
Modified: hive/branches/vectorization/contrib/src/java/org/apache/hadoop/hive/contrib/genericudf/example/GenericUDFDBOutput.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/contrib/src/java/org/apache/hadoop/hive/contrib/genericudf/example/GenericUDFDBOutput.java?rev=1514554&r1=1514553&r2=1514554&view=diff
==============================================================================
--- hive/branches/vectorization/contrib/src/java/org/apache/hadoop/hive/contrib/genericudf/example/GenericUDFDBOutput.java (original)
+++ hive/branches/vectorization/contrib/src/java/org/apache/hadoop/hive/contrib/genericudf/example/GenericUDFDBOutput.java Fri Aug 16 01:21:54 2013
@@ -62,12 +62,11 @@ import org.apache.hadoop.io.IntWritable;
+ "passed to the PreparedStatement object\n")
@UDFType(deterministic = false)
public class GenericUDFDBOutput extends GenericUDF {
- private static Log LOG = LogFactory
+ private static final Log LOG = LogFactory
.getLog(GenericUDFDBOutput.class.getName());
- ObjectInspector[] argumentOI;
- GenericUDFUtils.ReturnObjectInspectorResolver returnOIResolver;
- Connection connection = null;
+ private transient ObjectInspector[] argumentOI;
+ private transient Connection connection = null;
private String url;
private String user;
private String pass;
Modified: hive/branches/vectorization/contrib/src/java/org/apache/hadoop/hive/contrib/udtf/example/GenericUDTFExplode2.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/contrib/src/java/org/apache/hadoop/hive/contrib/udtf/example/GenericUDTFExplode2.java?rev=1514554&r1=1514553&r2=1514554&view=diff
==============================================================================
--- hive/branches/vectorization/contrib/src/java/org/apache/hadoop/hive/contrib/udtf/example/GenericUDTFExplode2.java (original)
+++ hive/branches/vectorization/contrib/src/java/org/apache/hadoop/hive/contrib/udtf/example/GenericUDTFExplode2.java Fri Aug 16 01:21:54 2013
@@ -38,7 +38,7 @@ import org.apache.hadoop.hive.serde2.obj
value = "_FUNC_(a) - like explode, but outputs two identical columns (for testing purposes)")
public class GenericUDTFExplode2 extends GenericUDTF {
- ListObjectInspector listOI = null;
+ private transient ListObjectInspector listOI = null;
@Override
public void close() throws HiveException {
Modified: hive/branches/vectorization/eclipse-templates/.classpath
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/eclipse-templates/.classpath?rev=1514554&r1=1514553&r2=1514554&view=diff
==============================================================================
--- hive/branches/vectorization/eclipse-templates/.classpath (original)
+++ hive/branches/vectorization/eclipse-templates/.classpath Fri Aug 16 01:21:54 2013
@@ -49,7 +49,6 @@
<classpathentry kind="lib" path="build/ivy/lib/default/avro-mapred-@avro.version@.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/default/jline-@jline.version@.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/default/json-@json.version@.jar"/>
- <classpathentry kind="lib" path="build/ivy/lib/default/asm-@asm.version@.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/default/commons-compress-@commons-compress.version@.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/default/commons-lang-@commons-lang.version@.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/default/commons-logging-@commons-logging.version@.jar"/>
Modified: hive/branches/vectorization/eclipse-templates/.classpath._hbase
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/eclipse-templates/.classpath._hbase?rev=1514554&r1=1514553&r2=1514554&view=diff
==============================================================================
--- hive/branches/vectorization/eclipse-templates/.classpath._hbase (original)
+++ hive/branches/vectorization/eclipse-templates/.classpath._hbase Fri Aug 16 01:21:54 2013
@@ -25,7 +25,6 @@
<classpathentry exported="true" kind="lib" path="build/hadoopcore/hadoop-@HADOOPVER@/lib/@JETTYUTILJAR@"/>
<classpathentry exported="true" kind="lib" path="cli/lib/jline-@jline.version@.jar"/>
<classpathentry exported="true" kind="lib" path="lib/json.jar"/>
- <classpathentry exported="true" kind="lib" path="lib/asm-@asm.version@.jar"/>
<classpathentry exported="true" kind="lib" path="lib/commons-cli-@commons-cli.version@.jar"/>
<classpathentry exported="true" kind="lib" path="lib/commons-codec-@commons-code.version@.jar"/>
<classpathentry exported="true" kind="lib" path="lib/commons-lang-@commons-lang.version@.jar"/>
Modified: hive/branches/vectorization/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java?rev=1514554&r1=1514553&r2=1514554&view=diff
==============================================================================
--- hive/branches/vectorization/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java (original)
+++ hive/branches/vectorization/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java Fri Aug 16 01:21:54 2013
@@ -109,6 +109,9 @@ public class HiveHBaseTableInputFormat e
Scan scan = new Scan();
boolean empty = true;
+ // The list of families that have been added to the scan
+ List<String> addedFamilies = new ArrayList<String>();
+
if (!addAll) {
for (int i : readColIDs) {
ColumnMapping colMap = columnsMapping.get(i);
@@ -118,8 +121,12 @@ public class HiveHBaseTableInputFormat e
if (colMap.qualifierName == null) {
scan.addFamily(colMap.familyNameBytes);
+ addedFamilies.add(colMap.familyName);
} else {
- scan.addColumn(colMap.familyNameBytes, colMap.qualifierNameBytes);
+ if(!addedFamilies.contains(colMap.familyName)){
+ // add only if the corresponding family has not already been added
+ scan.addColumn(colMap.familyNameBytes, colMap.qualifierNameBytes);
+ }
}
empty = false;
@@ -458,6 +465,9 @@ public class HiveHBaseTableInputFormat e
Scan scan = new Scan();
+ // The list of families that have been added to the scan
+ List<String> addedFamilies = new ArrayList<String>();
+
// REVIEW: are we supposed to be applying the getReadColumnIDs
// same as in getRecordReader?
for (int i = 0; i <columnsMapping.size(); i++) {
@@ -468,8 +478,12 @@ public class HiveHBaseTableInputFormat e
if (colMap.qualifierName == null) {
scan.addFamily(colMap.familyNameBytes);
+ addedFamilies.add(colMap.familyName);
} else {
- scan.addColumn(colMap.familyNameBytes, colMap.qualifierNameBytes);
+ if(!addedFamilies.contains(colMap.familyName)){
+ // add the column only if the family has not already been added
+ scan.addColumn(colMap.familyNameBytes, colMap.qualifierNameBytes);
+ }
}
}
Modified: hive/branches/vectorization/hbase-handler/src/test/templates/TestHBaseCliDriver.vm
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hbase-handler/src/test/templates/TestHBaseCliDriver.vm?rev=1514554&r1=1514553&r2=1514554&view=diff
==============================================================================
--- hive/branches/vectorization/hbase-handler/src/test/templates/TestHBaseCliDriver.vm (original)
+++ hive/branches/vectorization/hbase-handler/src/test/templates/TestHBaseCliDriver.vm Fri Aug 16 01:21:54 2013
@@ -26,10 +26,6 @@ import java.util.*;
import org.apache.hadoop.hive.hbase.HBaseQTestUtil;
import org.apache.hadoop.hive.hbase.HBaseTestSetup;
-import org.apache.hadoop.hive.ql.history.HiveHistoryViewer;
-import org.apache.hadoop.hive.ql.history.HiveHistory.QueryInfo;
-import org.apache.hadoop.hive.ql.history.HiveHistory.Keys;
-import org.apache.hadoop.hive.ql.history.HiveHistory.TaskInfo;
import org.apache.hadoop.hive.ql.session.SessionState;
public class $className extends TestCase {
@@ -114,21 +110,6 @@ public class $className extends TestCase
if (ecode != 0) {
fail("Client Execution failed with error code = " + ecode);
}
- if (SessionState.get() != null) {
- HiveHistoryViewer hv = new HiveHistoryViewer(SessionState.get()
- .getHiveHistory().getHistFileName());
- Map<String, QueryInfo> jobInfoMap = hv.getJobInfoMap();
- Map<String, TaskInfo> taskInfoMap = hv.getTaskInfoMap();
-
- if (jobInfoMap.size() != 0) {
- String cmd = (String)jobInfoMap.keySet().toArray()[0];
- QueryInfo ji = jobInfoMap.get(cmd);
-
- if (!ji.hm.get(Keys.QUERY_RET_CODE.name()).equals("0")) {
- fail("Wrong return code in hive history");
- }
- }
- }
ecode = qt.checkCliDriverResults(fname);
if (ecode != 0) {
Modified: hive/branches/vectorization/hcatalog/bin/hcat
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/bin/hcat?rev=1514554&r1=1514553&r2=1514554&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/bin/hcat (original)
+++ hive/branches/vectorization/hcatalog/bin/hcat Fri Aug 16 01:21:54 2013
@@ -78,7 +78,7 @@ fi
HIVE_HOME=${HIVE_HOME:-$HIVE_DIR}
#if hive is not in path and not set by env, set it to default in build tree
-if [ -n ${HIVE_HOME} ]; then
+if [ ! -n "${HIVE_HOME}" ]; then
HIVE_HOME="${bin}/../.."
echoerr "HIVE_HOME is not defined; assuming ${HIVE_HOME}";
fi
Modified: hive/branches/vectorization/hcatalog/build-support/ant/deploy.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/build-support/ant/deploy.xml?rev=1514554&r1=1514553&r2=1514554&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/build-support/ant/deploy.xml (original)
+++ hive/branches/vectorization/hcatalog/build-support/ant/deploy.xml Fri Aug 16 01:21:54 2013
@@ -81,9 +81,11 @@
<antcall target="hive-mvn-publish" />
<artifact:dependencies>
<dependency groupId="org.apache.hcatalog" artifactId="hcatalog" version="${hcatalog.version}" scope="system" systemPath="${path.to.basedir}/pom.xml"/>
+ <artifact:localRepository path="${mvn.local.repo}"/>
</artifact:dependencies>
<artifact:pom id="mvn.pom" file="pom.xml">
<profile id="${_mvn.hadoop.profile}"/>
+ <artifact:localRepository path="${mvn.local.repo}"/>
</artifact:pom>
<property name="mvn-init.complete" value="true"/>
</target>
Modified: hive/branches/vectorization/hcatalog/build-support/ant/test.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/build-support/ant/test.xml?rev=1514554&r1=1514553&r2=1514554&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/build-support/ant/test.xml (original)
+++ hive/branches/vectorization/hcatalog/build-support/ant/test.xml Fri Aug 16 01:21:54 2013
@@ -54,6 +54,7 @@
<path refid="test.class.path"/>
<pathelement location="${clover.jar}"/>
</classpath>
+ <jvmarg line="${junit.jvm.args}"/>
<formatter type="${test.junit.output.format}"/>
<batchtest fork="yes" todir="${test.logs}" unless="testcase">
<fileset dir="@{srcDir}" includes="**/Test*.java"/>
Modified: hive/branches/vectorization/hcatalog/build.properties
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/build.properties?rev=1514554&r1=1514553&r2=1514554&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/build.properties (original)
+++ hive/branches/vectorization/hcatalog/build.properties Fri Aug 16 01:21:54 2013
@@ -47,6 +47,9 @@ clover.db.dir=${build.dir}/test/clover/d
clover.report.dir=${build.dir}/test/clover/reports
clover.pdf.report.dir=${build.dir}/test/clover/pdf/reports
+# junit jvm args
+junit.jvm.args=-XX:-UseSplitVerifier -XX:+CMSClassUnloadingEnabled -XX:MaxPermSize=128M
+
apache-rat.version=0.8
findbugs.out.dir=${test.dir}/findbugs
Modified: hive/branches/vectorization/hcatalog/build.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/build.xml?rev=1514554&r1=1514553&r2=1514554&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/build.xml (original)
+++ hive/branches/vectorization/hcatalog/build.xml Fri Aug 16 01:21:54 2013
@@ -222,7 +222,7 @@
</copy>
</target>
- <target name="javadoc" depends="jar" description="Generate Javadoc documentation">
+ <target name="javadoc" depends="jar" description="Generate Javadoc documentation" unless="skip.javadoc">
<mkdir dir="${build.javadoc}"/>
<record name="${build.dir}/javadoc.log" action="start"/>
<javadoc overview="${src.dir}/../docs/overview.html"
Modified: hive/branches/vectorization/hcatalog/core/src/main/java/org/apache/hcatalog/mapreduce/FosterStorageHandler.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/core/src/main/java/org/apache/hcatalog/mapreduce/FosterStorageHandler.java?rev=1514554&r1=1514553&r2=1514554&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/core/src/main/java/org/apache/hcatalog/mapreduce/FosterStorageHandler.java (original)
+++ hive/branches/vectorization/hcatalog/core/src/main/java/org/apache/hcatalog/mapreduce/FosterStorageHandler.java Fri Aug 16 01:21:54 2013
@@ -117,9 +117,11 @@ public class FosterStorageHandler extend
String outputLocation;
- if (Boolean.valueOf((String)tableDesc.getProperties().get("EXTERNAL"))
+ if ((dynHash == null)
+ && Boolean.valueOf((String)tableDesc.getProperties().get("EXTERNAL"))
&& jobInfo.getLocation() != null && jobInfo.getLocation().length() > 0) {
- // honor external table that specifies the location
+ // honor custom location for external table apart from what metadata specifies
+ // only if we're not using dynamic partitioning - see HIVE-5011
outputLocation = jobInfo.getLocation();
} else if (dynHash == null && jobInfo.getPartitionValues().size() == 0) {
// For non-partitioned tables, we send them to the temp dir
Modified: hive/branches/vectorization/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/HCatMapReduceTest.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/HCatMapReduceTest.java?rev=1514554&r1=1514553&r2=1514554&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/HCatMapReduceTest.java (original)
+++ hive/branches/vectorization/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/HCatMapReduceTest.java Fri Aug 16 01:21:54 2013
@@ -34,6 +34,7 @@ import org.apache.hadoop.fs.LocalFileSys
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
+import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.SerDeInfo;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
@@ -84,6 +85,10 @@ public abstract class HCatMapReduceTest
private static FileSystem fs;
+ protected Boolean isTableExternal() {
+ return false;
+ }
+
protected String inputFormat() {
return RCFileInputFormat.class.getName();
}
@@ -137,7 +142,11 @@ public abstract class HCatMapReduceTest
Table tbl = new Table();
tbl.setDbName(databaseName);
tbl.setTableName(tableName);
- tbl.setTableType("MANAGED_TABLE");
+ if (isTableExternal()){
+ tbl.setTableType(TableType.EXTERNAL_TABLE.toString());
+ } else {
+ tbl.setTableType(TableType.MANAGED_TABLE.toString());
+ }
StorageDescriptor sd = new StorageDescriptor();
sd.setCols(getTableColumns());
@@ -150,6 +159,9 @@ public abstract class HCatMapReduceTest
sd.getSerdeInfo().setName(tbl.getTableName());
sd.getSerdeInfo().setParameters(new HashMap<String, String>());
sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_FORMAT, "1");
+ if (isTableExternal()){
+ sd.getSerdeInfo().getParameters().put("EXTERNAL", "TRUE");
+ }
sd.getSerdeInfo().setSerializationLib(serdeClass());
sd.setInputFormat(inputFormat());
sd.setOutputFormat(outputFormat());
Modified: hive/branches/vectorization/hcatalog/storage-handlers/hbase/build.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/storage-handlers/hbase/build.xml?rev=1514554&r1=1514553&r2=1514554&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/storage-handlers/hbase/build.xml (original)
+++ hive/branches/vectorization/hcatalog/storage-handlers/hbase/build.xml Fri Aug 16 01:21:54 2013
@@ -144,6 +144,7 @@
<path refid="test.class.path"/>
<pathelement path="${clover.jar}"/>
</classpath>
+ <jvmarg line="${junit.jvm.args}"/>
<formatter type="${test.junit.output.format}" />
<!-- If the user has not defined a particular test to run, run them all -->
<batchtest fork="yes" todir="${test.log.dir}" unless="testcase">
Modified: hive/branches/vectorization/hcatalog/webhcat/svr/build.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/webhcat/svr/build.xml?rev=1514554&r1=1514553&r2=1514554&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/webhcat/svr/build.xml (original)
+++ hive/branches/vectorization/hcatalog/webhcat/svr/build.xml Fri Aug 16 01:21:54 2013
@@ -41,14 +41,17 @@
<fileset dir="${build.dir}/lib/compile"/>
</path>
- <target name="compile-resource">
+ <target name="compile-resource" depends="javadoc">
<echo message="${ant.project.name}"/>
<property name="resources.dir" location="${path.to.basedir}/conf"/>
<copy todir="${build.classes}/">
<fileset dir="${basedir}/src/main/config/"/>
</copy>
+ </target>
+
+ <target name="javadoc" unless="skip.javadoc">
<echo message="Generating wadl xml file"/>
- <javadoc access="public" classpathref="compile.class.path">
+ <javadoc access="public" classpathref="compile.class.path" useexternalfile="yes">
<fileset dir="${basedir}/src/main/java" defaultexcludes="yes"/>
<doclet name="com.sun.jersey.wadl.resourcedoc.ResourceDoclet"
pathref="compile.class.path">
Modified: hive/branches/vectorization/ivy/libraries.properties
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ivy/libraries.properties?rev=1514554&r1=1514553&r2=1514554&view=diff
==============================================================================
--- hive/branches/vectorization/ivy/libraries.properties (original)
+++ hive/branches/vectorization/ivy/libraries.properties Fri Aug 16 01:21:54 2013
@@ -67,3 +67,4 @@ velocity.version=1.5
metrics-core.version=2.1.2
zookeeper.version=3.4.3
javolution.version=5.5.1
+mina.version=2.0.0-M5
Modified: hive/branches/vectorization/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java?rev=1514554&r1=1514553&r2=1514554&view=diff
==============================================================================
--- hive/branches/vectorization/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java (original)
+++ hive/branches/vectorization/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java Fri Aug 16 01:21:54 2013
@@ -33,6 +33,7 @@ import java.sql.SQLXML;
import java.sql.Savepoint;
import java.sql.Statement;
import java.sql.Struct;
+import java.util.HashMap;
import java.util.concurrent.Executor;
import java.util.LinkedList;
import java.util.List;
@@ -40,11 +41,13 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
+import javax.security.sasl.Sasl;
import javax.security.sasl.SaslException;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hive.service.auth.KerberosSaslHelper;
import org.apache.hive.service.auth.PlainSaslHelper;
+import org.apache.hive.service.auth.SaslQOP;
import org.apache.hive.service.cli.thrift.EmbeddedThriftCLIService;
import org.apache.hive.service.cli.thrift.TCLIService;
import org.apache.hive.service.cli.thrift.TCloseSessionReq;
@@ -65,6 +68,7 @@ import org.apache.thrift.transport.TTran
*/
public class HiveConnection implements java.sql.Connection {
private static final String HIVE_AUTH_TYPE= "auth";
+ private static final String HIVE_AUTH_QOP = "sasl.qop";
private static final String HIVE_AUTH_SIMPLE = "noSasl";
private static final String HIVE_AUTH_USER = "user";
private static final String HIVE_AUTH_PRINCIPAL = "principal";
@@ -132,8 +136,19 @@ public class HiveConnection implements j
|| !sessConf.get(HIVE_AUTH_TYPE).equals(HIVE_AUTH_SIMPLE)){
try {
if (sessConf.containsKey(HIVE_AUTH_PRINCIPAL)) {
+ Map<String, String> saslProps = new HashMap<String, String>();
+ SaslQOP saslQOP = SaslQOP.AUTH;
+ if(sessConf.containsKey(HIVE_AUTH_QOP)) {
+ try {
+ saslQOP = SaslQOP.fromString(sessConf.get(HIVE_AUTH_QOP));
+ } catch (IllegalArgumentException e) {
+ throw new SQLException("Invalid " + HIVE_AUTH_QOP + " parameter. " + e.getMessage(), "42000", e);
+ }
+ }
+ saslProps.put(Sasl.QOP, saslQOP.toString());
+ saslProps.put(Sasl.SERVER_AUTH, "true");
transport = KerberosSaslHelper.getKerberosTransport(
- sessConf.get(HIVE_AUTH_PRINCIPAL), host, transport);
+ sessConf.get(HIVE_AUTH_PRINCIPAL), host, transport, saslProps);
} else {
String userName = sessConf.get(HIVE_AUTH_USER);
if ((userName == null) || userName.isEmpty()) {
Modified: hive/branches/vectorization/jdbc/src/java/org/apache/hive/jdbc/HivePreparedStatement.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/jdbc/src/java/org/apache/hive/jdbc/HivePreparedStatement.java?rev=1514554&r1=1514553&r2=1514554&view=diff
==============================================================================
--- hive/branches/vectorization/jdbc/src/java/org/apache/hive/jdbc/HivePreparedStatement.java (original)
+++ hive/branches/vectorization/jdbc/src/java/org/apache/hive/jdbc/HivePreparedStatement.java Fri Aug 16 01:21:54 2013
@@ -25,7 +25,6 @@ import java.net.URL;
import java.sql.Array;
import java.sql.Blob;
import java.sql.Clob;
-import java.sql.Connection;
import java.sql.Date;
import java.sql.NClob;
import java.sql.ParameterMetaData;
@@ -35,71 +34,30 @@ import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.RowId;
import java.sql.SQLException;
-import java.sql.SQLWarning;
import java.sql.SQLXML;
import java.sql.Time;
import java.sql.Timestamp;
import java.util.Calendar;
import java.util.HashMap;
-import java.util.Map;
import org.apache.hive.service.cli.thrift.TCLIService;
-import org.apache.hive.service.cli.thrift.TExecuteStatementReq;
-import org.apache.hive.service.cli.thrift.TExecuteStatementResp;
-import org.apache.hive.service.cli.thrift.TOperationHandle;
import org.apache.hive.service.cli.thrift.TSessionHandle;
/**
* HivePreparedStatement.
*
*/
-public class HivePreparedStatement implements PreparedStatement {
+public class HivePreparedStatement extends HiveStatement implements PreparedStatement {
private final String sql;
- private TCLIService.Iface client;
- private final TSessionHandle sessHandle;
- private TOperationHandle stmtHandle;
- Map<String,String> sessConf = new HashMap<String,String>();
/**
* save the SQL parameters {paramLoc:paramValue}
*/
private final HashMap<Integer, String> parameters=new HashMap<Integer, String>();
- /**
- * We need to keep a reference to the result set to support the following:
- * <code>
- * statement.execute(String sql);
- * statement.getResultSet();
- * </code>.
- */
- private ResultSet resultSet = null;
- /**
- * The maximum number of rows this statement should return (0 => all rows).
- */
- private int maxRows = 0;
-
- /**
- * Add SQLWarnings to the warningChain if needed.
- */
- private SQLWarning warningChain = null;
-
- /**
- * Keep state so we can fail certain calls made after close().
- */
- private boolean isClosed = false;
-
- /**
- * keep the current ResultRet update count
- */
- private final int updateCount=0;
-
- /**
- *
- */
public HivePreparedStatement(TCLIService.Iface client, TSessionHandle sessHandle,
String sql) {
- this.client = client;
- this.sessHandle = sessHandle;
+ super(client, sessHandle);
this.sql = sql;
}
@@ -134,8 +92,7 @@ public class HivePreparedStatement imple
*/
public boolean execute() throws SQLException {
- ResultSet rs = executeImmediate(sql);
- return rs != null;
+ return super.execute(updateSql(sql, parameters));
}
/**
@@ -146,7 +103,7 @@ public class HivePreparedStatement imple
*/
public ResultSet executeQuery() throws SQLException {
- return executeImmediate(sql);
+ return super.executeQuery(updateSql(sql, parameters));
}
/*
@@ -156,45 +113,8 @@ public class HivePreparedStatement imple
*/
public int executeUpdate() throws SQLException {
- executeImmediate(sql);
- return updateCount;
- }
-
- /**
- * Executes the SQL statement.
- *
- * @param sql The sql, as a string, to execute
- * @return ResultSet
- * @throws SQLException if the prepared statement is closed or there is a database error.
- * caught Exceptions are thrown as SQLExceptions with the description
- * "08S01".
- */
-
- protected ResultSet executeImmediate(String sql) throws SQLException {
- if (isClosed) {
- throw new SQLException("Can't execute after statement has been closed");
- }
-
- try {
- clearWarnings();
- resultSet = null;
- if (sql.contains("?")) {
- sql = updateSql(sql, parameters);
- }
- TExecuteStatementReq execReq = new TExecuteStatementReq(sessHandle, sql);
- execReq.setConfOverlay(sessConf);
- TExecuteStatementResp execResp = client.ExecuteStatement(execReq);
- Utils.verifySuccessWithInfo(execResp.getStatus());
- stmtHandle = execResp.getOperationHandle();
- } catch (SQLException es) {
- throw es;
- } catch (Exception ex) {
- throw new SQLException(ex.toString(), "08S01", ex);
- }
- resultSet = new HiveQueryResultSet.Builder().setClient(client).setSessionHandle(sessHandle)
- .setStmtHandle(stmtHandle).setMaxRows(maxRows)
- .build();
- return resultSet;
+ super.executeUpdate(updateSql(sql, parameters));
+ return 0;
}
/**
@@ -205,6 +125,9 @@ public class HivePreparedStatement imple
* @return updated SQL string
*/
private String updateSql(final String sql, HashMap<Integer, String> parameters) {
+ if (!sql.contains("?")) {
+ return sql;
+ }
StringBuffer newSql = new StringBuffer(sql);
@@ -813,477 +736,4 @@ public class HivePreparedStatement imple
// TODO Auto-generated method stub
throw new SQLException("Method not supported");
}
-
- /*
- * (non-Javadoc)
- *
- * @see java.sql.Statement#addBatch(java.lang.String)
- */
-
- public void addBatch(String sql) throws SQLException {
- // TODO Auto-generated method stub
- throw new SQLException("Method not supported");
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.sql.Statement#cancel()
- */
-
- public void cancel() throws SQLException {
- // TODO Auto-generated method stub
- throw new SQLException("Method not supported");
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.sql.Statement#clearBatch()
- */
-
- public void clearBatch() throws SQLException {
- // TODO Auto-generated method stub
- throw new SQLException("Method not supported");
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.sql.Statement#clearWarnings()
- */
-
- public void clearWarnings() throws SQLException {
- warningChain=null;
- }
-
- public void closeOnCompletion() throws SQLException {
- // JDK 1.7
- throw new SQLException("Method not supported");
- }
-
- /**
- * Closes the prepared statement.
- *
- * @throws SQLException
- */
-
- public void close() throws SQLException {
- client = null;
- if (resultSet!=null) {
- resultSet.close();
- resultSet = null;
- }
- isClosed = true;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.sql.Statement#execute(java.lang.String)
- */
-
- public boolean execute(String sql) throws SQLException {
- // TODO Auto-generated method stub
- throw new SQLException("Method not supported");
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.sql.Statement#execute(java.lang.String, int)
- */
-
- public boolean execute(String sql, int autoGeneratedKeys) throws SQLException {
- // TODO Auto-generated method stub
- throw new SQLException("Method not supported");
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.sql.Statement#execute(java.lang.String, int[])
- */
-
- public boolean execute(String sql, int[] columnIndexes) throws SQLException {
- // TODO Auto-generated method stub
- throw new SQLException("Method not supported");
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.sql.Statement#execute(java.lang.String, java.lang.String[])
- */
-
- public boolean execute(String sql, String[] columnNames) throws SQLException {
- // TODO Auto-generated method stub
- throw new SQLException("Method not supported");
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.sql.Statement#executeBatch()
- */
-
- public int[] executeBatch() throws SQLException {
- // TODO Auto-generated method stub
- throw new SQLException("Method not supported");
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.sql.Statement#executeQuery(java.lang.String)
- */
-
- public ResultSet executeQuery(String sql) throws SQLException {
- // TODO Auto-generated method stub
- throw new SQLException("Method not supported");
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.sql.Statement#executeUpdate(java.lang.String)
- */
-
- public int executeUpdate(String sql) throws SQLException {
- // TODO Auto-generated method stub
- throw new SQLException("Method not supported");
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.sql.Statement#executeUpdate(java.lang.String, int)
- */
-
- public int executeUpdate(String sql, int autoGeneratedKeys) throws SQLException {
- // TODO Auto-generated method stub
- throw new SQLException("Method not supported");
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.sql.Statement#executeUpdate(java.lang.String, int[])
- */
-
- public int executeUpdate(String sql, int[] columnIndexes) throws SQLException {
- // TODO Auto-generated method stub
- throw new SQLException("Method not supported");
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.sql.Statement#executeUpdate(java.lang.String, java.lang.String[])
- */
-
- public int executeUpdate(String sql, String[] columnNames) throws SQLException {
- // TODO Auto-generated method stub
- throw new SQLException("Method not supported");
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.sql.Statement#getConnection()
- */
-
- public Connection getConnection() throws SQLException {
- // TODO Auto-generated method stub
- throw new SQLException("Method not supported");
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.sql.Statement#getFetchDirection()
- */
-
- public int getFetchDirection() throws SQLException {
- // TODO Auto-generated method stub
- throw new SQLException("Method not supported");
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.sql.Statement#getFetchSize()
- */
-
- public int getFetchSize() throws SQLException {
- // TODO Auto-generated method stub
- throw new SQLException("Method not supported");
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.sql.Statement#getGeneratedKeys()
- */
-
- public ResultSet getGeneratedKeys() throws SQLException {
- // TODO Auto-generated method stub
- throw new SQLException("Method not supported");
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.sql.Statement#getMaxFieldSize()
- */
-
- public int getMaxFieldSize() throws SQLException {
- // TODO Auto-generated method stub
- throw new SQLException("Method not supported");
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.sql.Statement#getMaxRows()
- */
-
- public int getMaxRows() throws SQLException {
- return this.maxRows;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.sql.Statement#getMoreResults()
- */
-
- public boolean getMoreResults() throws SQLException {
- // TODO Auto-generated method stub
- throw new SQLException("Method not supported");
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.sql.Statement#getMoreResults(int)
- */
-
- public boolean getMoreResults(int current) throws SQLException {
- // TODO Auto-generated method stub
- throw new SQLException("Method not supported");
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.sql.Statement#getQueryTimeout()
- */
-
- public int getQueryTimeout() throws SQLException {
- // TODO Auto-generated method stub
- throw new SQLException("Method not supported");
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.sql.Statement#getResultSet()
- */
-
- public ResultSet getResultSet() throws SQLException {
- return this.resultSet;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.sql.Statement#getResultSetConcurrency()
- */
-
- public int getResultSetConcurrency() throws SQLException {
- // TODO Auto-generated method stub
- throw new SQLException("Method not supported");
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.sql.Statement#getResultSetHoldability()
- */
-
- public int getResultSetHoldability() throws SQLException {
- // TODO Auto-generated method stub
- throw new SQLException("Method not supported");
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.sql.Statement#getResultSetType()
- */
-
- public int getResultSetType() throws SQLException {
- // TODO Auto-generated method stub
- throw new SQLException("Method not supported");
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.sql.Statement#getUpdateCount()
- */
-
- public int getUpdateCount() throws SQLException {
- return updateCount;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.sql.Statement#getWarnings()
- */
-
- public SQLWarning getWarnings() throws SQLException {
- return warningChain;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.sql.Statement#isClosed()
- */
-
- public boolean isClosed() throws SQLException {
- return isClosed;
- }
-
- public boolean isCloseOnCompletion() throws SQLException {
- //JDK 1.7
- throw new SQLException("Method not supported");
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.sql.Statement#isPoolable()
- */
-
- public boolean isPoolable() throws SQLException {
- // TODO Auto-generated method stub
- throw new SQLException("Method not supported");
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.sql.Statement#setCursorName(java.lang.String)
- */
-
- public void setCursorName(String name) throws SQLException {
- // TODO Auto-generated method stub
- throw new SQLException("Method not supported");
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.sql.Statement#setEscapeProcessing(boolean)
- */
-
- public void setEscapeProcessing(boolean enable) throws SQLException {
- // TODO Auto-generated method stub
- throw new SQLException("Method not supported");
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.sql.Statement#setFetchDirection(int)
- */
-
- public void setFetchDirection(int direction) throws SQLException {
- // TODO Auto-generated method stub
- throw new SQLException("Method not supported");
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.sql.Statement#setFetchSize(int)
- */
-
- public void setFetchSize(int rows) throws SQLException {
- // TODO Auto-generated method stub
- throw new SQLException("Method not supported");
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.sql.Statement#setMaxFieldSize(int)
- */
-
- public void setMaxFieldSize(int max) throws SQLException {
- // TODO Auto-generated method stub
- throw new SQLException("Method not supported");
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.sql.Statement#setMaxRows(int)
- */
-
- public void setMaxRows(int max) throws SQLException {
- if (max < 0) {
- throw new SQLException("max must be >= 0");
- }
- this.maxRows = max;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.sql.Statement#setPoolable(boolean)
- */
-
- public void setPoolable(boolean poolable) throws SQLException {
- // TODO Auto-generated method stub
- throw new SQLException("Method not supported");
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.sql.Statement#setQueryTimeout(int)
- */
-
- public void setQueryTimeout(int seconds) throws SQLException {
- // TODO Auto-generated method stub
- // throw new SQLException("Method not supported");
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.sql.Wrapper#isWrapperFor(java.lang.Class)
- */
-
- public boolean isWrapperFor(Class<?> iface) throws SQLException {
- // TODO Auto-generated method stub
- throw new SQLException("Method not supported");
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.sql.Wrapper#unwrap(java.lang.Class)
- */
-
- public <T> T unwrap(Class<T> iface) throws SQLException {
- // TODO Auto-generated method stub
- throw new SQLException("Method not supported");
- }
-
}
Modified: hive/branches/vectorization/jdbc/src/test/org/apache/hive/jdbc/TestJdbcDriver2.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/jdbc/src/test/org/apache/hive/jdbc/TestJdbcDriver2.java?rev=1514554&r1=1514553&r2=1514554&view=diff
==============================================================================
--- hive/branches/vectorization/jdbc/src/test/org/apache/hive/jdbc/TestJdbcDriver2.java (original)
+++ hive/branches/vectorization/jdbc/src/test/org/apache/hive/jdbc/TestJdbcDriver2.java Fri Aug 16 01:21:54 2013
@@ -42,7 +42,12 @@ import junit.framework.TestCase;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hive.common.util.HiveVersionInfo;
+import org.apache.hive.service.cli.operation.ClassicTableTypeMapping;
+import org.apache.hive.service.cli.operation.ClassicTableTypeMapping.ClassicTableTypes;
+import org.apache.hive.service.cli.operation.HiveTableTypeMapping;
+import org.apache.hive.service.cli.operation.TableTypeMappingFactory.TableTypeMappings;
/**
@@ -384,6 +389,46 @@ public class TestJdbcDriver2 extends Tes
expectedException);
}
+ /**
+ * Execute non-select statements using execute() and executeUpdated() APIs
+ * of PreparedStatement interface
+ * @throws Exception
+ */
+ public void testExecutePreparedStatement() throws Exception {
+ String key = "testKey";
+ String val1 = "val1";
+ String val2 = "val2";
+ PreparedStatement ps = con.prepareStatement("set " + key + " = ?");
+
+ // execute() of Prepared statement
+ ps.setString(1, val1);
+ ps.execute();
+ verifyConfValue(key, val1);
+
+ // executeUpdate() of Prepared statement
+ ps.clearParameters();
+ ps.setString(1, val2);
+ ps.executeUpdate();
+ verifyConfValue(key, val2);
+ }
+
+ /**
+ * Execute "set x" and extract value from key=val format result
+ * Verify the extracted value
+ * @param stmt
+ * @return
+ * @throws Exception
+ */
+ private void verifyConfValue(String key, String expectedVal) throws Exception {
+ Statement stmt = con.createStatement();
+ ResultSet res = stmt.executeQuery("set " + key);
+ assertTrue(res.next());
+ String resultValues[] = res.getString(1).split("="); // "key = 'val'"
+ assertEquals("Result not in key = val format", 2, resultValues.length);
+ String result = resultValues[1].substring(1, resultValues[1].length() -1); // remove '
+ assertEquals("Conf value should be set by execute()", expectedVal, result);
+ }
+
public final void testSelectAll() throws Exception {
doTestSelectAll(tableName, -1, -1); // tests not setting maxRows (return all)
doTestSelectAll(tableName, 0, -1); // tests setting maxRows to 0 (return all)
@@ -664,6 +709,25 @@ public class TestJdbcDriver2 extends Tes
}
public void testMetaDataGetTables() throws SQLException {
+ getTablesTest(TableType.MANAGED_TABLE.toString(), TableType.VIRTUAL_VIEW.toString());
+ }
+
+ public void testMetaDataGetTablesHive() throws SQLException {
+ Statement stmt = con.createStatement();
+ stmt.execute("set " + HiveConf.ConfVars.HIVE_SERVER2_TABLE_TYPE_MAPPING.varname +
+ " = " + TableTypeMappings.HIVE.toString());
+ getTablesTest(TableType.MANAGED_TABLE.toString(), TableType.VIRTUAL_VIEW.toString());
+ }
+
+ public void testMetaDataGetTablesClassic() throws SQLException {
+ Statement stmt = con.createStatement();
+ stmt.execute("set " + HiveConf.ConfVars.HIVE_SERVER2_TABLE_TYPE_MAPPING.varname +
+ " = " + TableTypeMappings.CLASSIC.toString());
+ stmt.close();
+ getTablesTest(ClassicTableTypes.TABLE.toString(), ClassicTableTypes.VIEW.toString());
+ }
+
+ private void getTablesTest(String tableTypeName, String viewTypeName) throws SQLException {
Map<String, Object[]> tests = new HashMap<String, Object[]>();
tests.put("test%jdbc%", new Object[]{"testhivejdbcdriver_table"
, "testhivejdbcdriverpartitionedtable"
@@ -698,7 +762,9 @@ public class TestJdbcDriver2 extends Tes
assertTrue("Missing comment on the table.", resultTableComment.length()>0);
String tableType = rs.getString("TABLE_TYPE");
if (resultTableName.endsWith("view")) {
- assertEquals("Expected a tabletype view but got something else.", "VIRTUAL_VIEW", tableType);
+ assertEquals("Expected a tabletype view but got something else.", viewTypeName, tableType);
+ } else {
+ assertEquals("Expected a tabletype table but got something else.", tableTypeName, tableType);
}
cnt++;
}
@@ -708,7 +774,7 @@ public class TestJdbcDriver2 extends Tes
// only ask for the views.
ResultSet rs = (ResultSet)con.getMetaData().getTables("default", null, null
- , new String[]{"VIRTUAL_VIEW"});
+ , new String[]{viewTypeName});
int cnt=0;
while (rs.next()) {
cnt++;
@@ -742,13 +808,28 @@ public class TestJdbcDriver2 extends Tes
}
public void testMetaDataGetTableTypes() throws SQLException {
- ResultSet rs = (ResultSet)con.getMetaData().getTableTypes();
+ metaDataGetTableTypeTest(new HiveTableTypeMapping().getTableTypeNames());
+ }
- Set<String> tabletypes = new HashSet();
- tabletypes.add("MANAGED_TABLE");
- tabletypes.add("EXTERNAL_TABLE");
- tabletypes.add("VIRTUAL_VIEW");
- tabletypes.add("INDEX_TABLE");
+ public void testMetaDataGetHiveTableTypes() throws SQLException {
+ Statement stmt = con.createStatement();
+ stmt.execute("set " + HiveConf.ConfVars.HIVE_SERVER2_TABLE_TYPE_MAPPING.varname +
+ " = " + TableTypeMappings.HIVE.toString());
+ stmt.close();
+ metaDataGetTableTypeTest(new HiveTableTypeMapping().getTableTypeNames());
+ }
+
+ public void testMetaDataGetClassicTableTypes() throws SQLException {
+ Statement stmt = con.createStatement();
+ stmt.execute("set " + HiveConf.ConfVars.HIVE_SERVER2_TABLE_TYPE_MAPPING.varname +
+ " = " + TableTypeMappings.CLASSIC.toString());
+ stmt.close();
+ metaDataGetTableTypeTest(new ClassicTableTypeMapping().getTableTypeNames());
+ }
+
+ private void metaDataGetTableTypeTest(Set<String> tabletypes)
+ throws SQLException {
+ ResultSet rs = (ResultSet)con.getMetaData().getTableTypes();
int cnt = 0;
while (rs.next()) {
Modified: hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java?rev=1514554&r1=1514553&r2=1514554&view=diff
==============================================================================
--- hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (original)
+++ hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java Fri Aug 16 01:21:54 2013
@@ -651,17 +651,7 @@ public class HiveMetaStore extends Thrif
success = true;
} catch (Exception e) {
ex = e;
- if (e instanceof MetaException) {
- throw (MetaException) e;
- } else if (e instanceof NoSuchObjectException) {
- throw (NoSuchObjectException) e;
- } else if (e instanceof TException) {
- throw (TException) e;
- } else {
- MetaException me = new MetaException(e.toString());
- me.initCause(e);
- throw me;
- }
+ rethrowException(e);
} finally {
endFunction("alter_database", success, ex);
}
@@ -2038,17 +2028,7 @@ public class HiveMetaStore extends Thrif
throw new MetaException(e.getMessage());
} catch (Exception e) {
ex = e;
- if (e instanceof MetaException) {
- throw (MetaException) e;
- } else if (e instanceof NoSuchObjectException) {
- throw (NoSuchObjectException) e;
- } else if (e instanceof TException) {
- throw (TException) e;
- } else {
- MetaException me = new MetaException(e.toString());
- me.initCause(e);
- throw me;
- }
+ rethrowException(e);
} finally {
endFunction("drop_partition", ret, ex, tbl_name);
}
@@ -2099,17 +2079,7 @@ public class HiveMetaStore extends Thrif
throw new NoSuchObjectException(e.getMessage());
} catch (Exception e) {
ex = e;
- if (e instanceof MetaException) {
- throw (MetaException) e;
- } else if (e instanceof NoSuchObjectException) {
- throw (NoSuchObjectException) e;
- } else if (e instanceof TException) {
- throw (TException) e;
- } else {
- MetaException me = new MetaException(e.toString());
- me.initCause(e);
- throw me;
- }
+ rethrowException(e);
} finally {
endFunction("get_partition_with_auth", ret != null, ex, tbl_name);
}
@@ -2159,17 +2129,7 @@ public class HiveMetaStore extends Thrif
throw new NoSuchObjectException(e.getMessage());
} catch (Exception e) {
ex = e;
- if (e instanceof MetaException) {
- throw (MetaException) e;
- } else if (e instanceof NoSuchObjectException) {
- throw (NoSuchObjectException) e;
- } else if (e instanceof TException) {
- throw (TException) e;
- } else {
- MetaException me = new MetaException(e.toString());
- me.initCause(e);
- throw me;
- }
+ rethrowException(e);
} finally {
endFunction("get_partitions_with_auth", ret != null, ex, tblName);
}
@@ -2714,17 +2674,7 @@ public class HiveMetaStore extends Thrif
ret = get_partition_by_name_core(getMS(), db_name, tbl_name, part_name);
} catch (Exception e) {
ex = e;
- if (e instanceof MetaException) {
- throw (MetaException) e;
- } else if (e instanceof NoSuchObjectException) {
- throw (NoSuchObjectException) e;
- } else if (e instanceof TException) {
- throw (TException) e;
- } else {
- MetaException me = new MetaException(e.toString());
- me.initCause(e);
- throw me;
- }
+ rethrowException(e);
} finally {
endFunction("get_partition_by_name", ret != null, ex, tbl_name);
}
@@ -2813,17 +2763,7 @@ public class HiveMetaStore extends Thrif
throw new MetaException(e.getMessage());
} catch (Exception e) {
ex = e;
- if (e instanceof MetaException) {
- throw (MetaException) e;
- } else if (e instanceof NoSuchObjectException) {
- throw (NoSuchObjectException) e;
- } else if (e instanceof TException) {
- throw (TException) e;
- } else {
- MetaException me = new MetaException(e.toString());
- me.initCause(e);
- throw me;
- }
+ rethrowException(e);
} finally {
endFunction("drop_partition_by_name", ret, ex, tbl_name);
}
@@ -2844,17 +2784,7 @@ public class HiveMetaStore extends Thrif
max_parts, null, null);
} catch (Exception e) {
ex = e;
- if (e instanceof MetaException) {
- throw (MetaException) e;
- } else if (e instanceof NoSuchObjectException) {
- throw (NoSuchObjectException) e;
- } else if (e instanceof TException) {
- throw (TException) e;
- } else {
- MetaException me = new MetaException(e.toString());
- me.initCause(e);
- throw me;
- }
+ rethrowException(e);
} finally {
endFunction("get_partitions_ps", ret != null, ex, tbl_name);
}
@@ -2879,17 +2809,7 @@ public class HiveMetaStore extends Thrif
throw new MetaException(e.getMessage());
} catch (Exception e) {
ex = e;
- if (e instanceof MetaException) {
- throw (MetaException) e;
- } else if (e instanceof NoSuchObjectException) {
- throw (NoSuchObjectException) e;
- } else if (e instanceof TException) {
- throw (TException) e;
- } else {
- MetaException me = new MetaException(e.toString());
- me.initCause(e);
- throw me;
- }
+ rethrowException(e);
} finally {
endFunction("get_partitions_ps_with_auth", ret != null, ex, tbl_name);
}
@@ -2907,17 +2827,7 @@ public class HiveMetaStore extends Thrif
ret = getMS().listPartitionNamesPs(db_name, tbl_name, part_vals, max_parts);
} catch (Exception e) {
ex = e;
- if (e instanceof MetaException) {
- throw (MetaException) e;
- } else if (e instanceof NoSuchObjectException) {
- throw (NoSuchObjectException) e;
- } else if (e instanceof TException) {
- throw (TException) e;
- } else {
- MetaException me = new MetaException(e.toString());
- me.initCause(e);
- throw me;
- }
+ rethrowException(e);
} finally {
endFunction("get_partitions_names_ps", ret != null, ex, tbl_name);
}
@@ -3049,17 +2959,7 @@ public class HiveMetaStore extends Thrif
throw new MetaException(e.getMessage());
} catch (Exception e) {
ex = e;
- if (e instanceof MetaException) {
- throw (MetaException) e;
- } else if (e instanceof NoSuchObjectException) {
- throw (NoSuchObjectException) e;
- } else if (e instanceof TException) {
- throw (TException) e;
- } else {
- MetaException me = new MetaException(e.toString());
- me.initCause(e);
- throw me;
- }
+ rethrowException(e);
} finally {
endFunction("drop_index_by_name", ret, ex, tblName);
}
@@ -3139,17 +3039,7 @@ public class HiveMetaStore extends Thrif
ret = get_index_by_name_core(getMS(), dbName, tblName, indexName);
} catch (Exception e) {
ex = e;
- if (e instanceof MetaException) {
- throw (MetaException) e;
- } else if (e instanceof NoSuchObjectException) {
- throw (NoSuchObjectException) e;
- } else if (e instanceof TException) {
- throw (TException) e;
- } else {
- MetaException me = new MetaException(e.toString());
- me.initCause(e);
- throw me;
- }
+ rethrowException(e);
} finally {
endFunction("drop_index_by_name", ret != null, ex, tblName);
}
@@ -3206,17 +3096,7 @@ public class HiveMetaStore extends Thrif
ret = getMS().getIndexes(dbName, tblName, maxIndexes);
} catch (Exception e) {
ex = e;
- if (e instanceof MetaException) {
- throw (MetaException) e;
- } else if (e instanceof NoSuchObjectException) {
- throw (NoSuchObjectException) e;
- } else if (e instanceof TException) {
- throw (TException) e;
- } else {
- MetaException me = new MetaException(e.toString());
- me.initCause(e);
- throw me;
- }
+ rethrowException(e);
} finally {
endFunction("get_indexes", ret != null, ex, tblName);
}
@@ -3424,23 +3304,29 @@ public class HiveMetaStore extends Thrif
ret = getMS().getPartitionsByFilter(dbName, tblName, filter, maxParts);
} catch (Exception e) {
ex = e;
- if (e instanceof MetaException) {
- throw (MetaException) e;
- } else if (e instanceof NoSuchObjectException) {
- throw (NoSuchObjectException) e;
- } else if (e instanceof TException) {
- throw (TException) e;
- } else {
- MetaException me = new MetaException(e.toString());
- me.initCause(e);
- throw me;
- }
+ rethrowException(e);
} finally {
endFunction("get_partitions_by_filter", ret != null, ex, tblName);
}
return ret;
}
+ private void rethrowException(Exception e)
+ throws MetaException, NoSuchObjectException, TException {
+ // TODO: Both of these are TException, why do we need these separate clauses?
+ if (e instanceof MetaException) {
+ throw (MetaException) e;
+ } else if (e instanceof NoSuchObjectException) {
+ throw (NoSuchObjectException) e;
+ } else if (e instanceof TException) {
+ throw (TException) e;
+ } else {
+ MetaException me = new MetaException(e.toString());
+ me.initCause(e);
+ throw me;
+ }
+ }
+
@Override
public List<Partition> get_partitions_by_names(final String dbName,
final String tblName, final List<String> partNames)
@@ -3454,17 +3340,7 @@ public class HiveMetaStore extends Thrif
ret = getMS().getPartitionsByNames(dbName, tblName, partNames);
} catch (Exception e) {
ex = e;
- if (e instanceof MetaException) {
- throw (MetaException) e;
- } else if (e instanceof NoSuchObjectException) {
- throw (NoSuchObjectException) e;
- } else if (e instanceof TException) {
- throw (TException) e;
- } else {
- MetaException me = new MetaException(e.toString());
- me.initCause(e);
- throw me;
- }
+ rethrowException(e);
} finally {
endFunction("get_partitions_by_names", ret != null, ex, tblName);
}
@@ -4376,7 +4252,8 @@ public class HiveMetaStore extends Thrif
// start delegation token manager
HMSHandler hmsHandler = new HMSHandler("new db based metaserver", conf);
saslServer.startDelegationTokenSecretManager(conf, hmsHandler);
- transFactory = saslServer.createTransportFactory();
+ transFactory = saslServer.createTransportFactory(
+ MetaStoreUtils.getMetaStoreSaslProperties(conf));
processor = saslServer.wrapProcessor(
new ThriftHiveMetastore.Processor<HMSHandler>(hmsHandler));
LOG.info("Starting DB backed MetaStore Server in Secure Mode");
Modified: hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java?rev=1514554&r1=1514553&r2=1514554&view=diff
==============================================================================
--- hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java (original)
+++ hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java Fri Aug 16 01:21:54 2013
@@ -258,17 +258,17 @@ public class HiveMetaStoreClient impleme
String tokenSig = conf.get("hive.metastore.token.signature");
// tokenSig could be null
tokenStrForm = shim.getTokenStrForm(tokenSig);
-
if(tokenStrForm != null) {
// authenticate using delegation tokens via the "DIGEST" mechanism
transport = authBridge.createClientTransport(null, store.getHost(),
- "DIGEST", tokenStrForm, transport);
+ "DIGEST", tokenStrForm, transport,
+ MetaStoreUtils.getMetaStoreSaslProperties(conf));
} else {
String principalConfig =
conf.getVar(HiveConf.ConfVars.METASTORE_KERBEROS_PRINCIPAL);
transport = authBridge.createClientTransport(
principalConfig, store.getHost(), "KERBEROS", null,
- transport);
+ transport, MetaStoreUtils.getMetaStoreSaslProperties(conf));
}
} catch (IOException ioe) {
LOG.error("Couldn't create client transport", ioe);
Modified: hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java?rev=1514554&r1=1514553&r2=1514554&view=diff
==============================================================================
--- hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java (original)
+++ hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java Fri Aug 16 01:21:54 2013
@@ -65,9 +65,9 @@ import org.apache.hadoop.hive.serde2.obj
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge;
-
public class MetaStoreUtils {
protected static final Log LOG = LogFactory.getLog("hive.log");
@@ -1265,6 +1265,18 @@ public class MetaStoreUtils {
return true;
}
+ /**
+ * Read and return the meta store Sasl configuration. Currently it uses the default
+ * Hadoop SASL configuration and can be configured using "hadoop.rpc.protection"
+ * @param conf
+ * @return The SASL configuration
+ */
+ public static Map<String, String> getMetaStoreSaslProperties(HiveConf conf) {
+ // As of now Hive Meta Store uses the same configuration as Hadoop SASL configuration
+ return ShimLoader.getHadoopThriftAuthBridge().getHadoopSaslProperties(conf);
+ }
+
+
private static String getPartitionValWithInvalidCharacter(List<String> partVals,
Pattern partitionValidationPattern) {
if (partitionValidationPattern == null) {
Modified: hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java?rev=1514554&r1=1514553&r2=1514554&view=diff
==============================================================================
--- hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (original)
+++ hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java Fri Aug 16 01:21:54 2013
@@ -19,6 +19,7 @@
package org.apache.hadoop.hive.metastore;
import static org.apache.commons.lang.StringUtils.join;
+import static org.apache.commons.lang.StringUtils.repeat;
import java.net.URI;
import java.net.URISyntaxException;
@@ -33,6 +34,7 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.Set;
+import java.util.TreeMap;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
@@ -157,6 +159,7 @@ public class ObjectStore implements RawS
private boolean isInitialized = false;
private PersistenceManager pm = null;
+ private MetaStoreDirectSql directSql = null;
private Configuration hiveConf;
int openTrasactionCalls = 0;
private Transaction currentTransaction = null;
@@ -196,6 +199,7 @@ public class ObjectStore implements RawS
// Always want to re-create pm as we don't know if it were created by the
// most recent instance of the pmf
pm = null;
+ directSql = null;
openTrasactionCalls = 0;
currentTransaction = null;
transactionStatus = TXN_STATUS.NO_STATE;
@@ -227,6 +231,9 @@ public class ObjectStore implements RawS
prop = dsProps;
pm = getPersistenceManager();
isInitialized = pm != null;
+ if (isInitialized) {
+ directSql = new MetaStoreDirectSql(pm);
+ }
return;
}
@@ -1652,43 +1659,34 @@ public class ObjectStore implements RawS
@Override
public List<Partition> getPartitionsByNames(String dbName, String tblName,
List<String> partNames) throws MetaException, NoSuchObjectException {
+ boolean doTrace = LOG.isDebugEnabled();
+ List<Partition> results = null;
+ boolean doUseDirectSql = HiveConf.getBoolVar(getConf(), ConfVars.METASTORE_TRY_DIRECT_SQL);
boolean success = false;
try {
+ long start = doTrace ? System.nanoTime() : 0;
openTransaction();
-
- StringBuilder sb = new StringBuilder(
- "table.tableName == t1 && table.database.name == t2 && (");
- int n = 0;
- Map<String, String> params = new HashMap<String, String>();
- for (Iterator<String> itr = partNames.iterator(); itr.hasNext();) {
- String pn = "p" + n;
- n++;
- String part = itr.next();
- params.put(pn, part);
- sb.append("partitionName == ").append(pn);
- sb.append(" || ");
+ if (doUseDirectSql) {
+ try {
+ results = directSql.getPartitionsViaSqlFilter(dbName, tblName, partNames);
+ } catch (Exception ex) {
+ LOG.error("Direct SQL failed, falling back to ORM", ex);
+ doUseDirectSql = false;
+ rollbackTransaction();
+ start = doTrace ? System.nanoTime() : 0;
+ openTransaction();
+ }
}
- sb.setLength(sb.length() - 4); // remove the last " || "
- sb.append(')');
-
- Query query = pm.newQuery(MPartition.class, sb.toString());
-
- LOG.debug(" JDOQL filter is " + sb.toString());
-
- params.put("t1", tblName.trim());
- params.put("t2", dbName.trim());
-
- String parameterDeclaration = makeParameterDeclarationString(params);
- query.declareParameters(parameterDeclaration);
- query.setOrdering("partitionName ascending");
- List<MPartition> mparts = (List<MPartition>) query.executeWithMap(params);
- // pm.retrieveAll(mparts); // retrieveAll is pessimistic. some fields may not be needed
- List<Partition> results = convertToParts(dbName, tblName, mparts);
- // pm.makeTransientAll(mparts); // makeTransient will prohibit future access of unfetched fields
- query.closeAll();
+ if (!doUseDirectSql) {
+ results = getPartitionsViaOrm(dbName, tblName, partNames);
+ }
success = commitTransaction();
+ if (doTrace) {
+ LOG.debug(results.size() + " partition retrieved using " + (doUseDirectSql ? "SQL" : "ORM")
+ + " in " + ((System.nanoTime() - start) / 1000000.0) + "ms");
+ }
return results;
} finally {
if (!success) {
@@ -1697,15 +1695,98 @@ public class ObjectStore implements RawS
}
}
+ private List<Partition> getPartitionsViaOrm(
+ String dbName, String tblName, List<String> partNames) throws MetaException {
+ StringBuilder sb = new StringBuilder(
+ "table.tableName == t1 && table.database.name == t2 && (");
+ int n = 0;
+ Map<String, String> params = new HashMap<String, String>();
+ for (Iterator<String> itr = partNames.iterator(); itr.hasNext();) {
+ String pn = "p" + n;
+ n++;
+ String part = itr.next();
+ params.put(pn, part);
+ sb.append("partitionName == ").append(pn);
+ sb.append(" || ");
+ }
+ sb.setLength(sb.length() - 4); // remove the last " || "
+ sb.append(')');
+
+ Query query = pm.newQuery(MPartition.class, sb.toString());
+
+ LOG.debug(" JDOQL filter is " + sb.toString());
+ params.put("t1", tblName.trim());
+ params.put("t2", dbName.trim());
+
+ String parameterDeclaration = makeParameterDeclarationString(params);
+
+ query.declareParameters(parameterDeclaration);
+ query.setOrdering("partitionName ascending");
+
+ List<MPartition> mparts = (List<MPartition>) query.executeWithMap(params);
+ // pm.retrieveAll(mparts); // retrieveAll is pessimistic. some fields may not be needed
+ List<Partition> results = convertToParts(dbName, tblName, mparts);
+ // pm.makeTransientAll(mparts); // makeTransient will prohibit future access of unfetched fields
+ query.closeAll();
+ return results;
+ }
+
@Override
public List<Partition> getPartitionsByFilter(String dbName, String tblName,
String filter, short maxParts) throws MetaException, NoSuchObjectException {
- openTransaction();
- List<Partition> parts = convertToParts(listMPartitionsByFilter(dbName,
- tblName, filter, maxParts));
- LOG.info("# parts after pruning = " + parts.size());
- commitTransaction();
- return parts;
+ boolean doTrace = LOG.isDebugEnabled();
+ // There's no portable SQL limit. It doesn't make a lot of sense w/o offset anyway.
+ boolean doUseDirectSql = (maxParts < 0)
+ && HiveConf.getBoolVar(getConf(), ConfVars.METASTORE_TRY_DIRECT_SQL);
+ dbName = dbName.toLowerCase();
+ tblName = tblName.toLowerCase();
+ List<Partition> results = null;
+ FilterParser parser = null;
+ if (filter != null && filter.length() != 0) {
+ LOG.debug("Filter specified is " + filter);
+ parser = getFilterParser(filter);
+ }
+
+ boolean success = false;
+ try {
+ long start = doTrace ? System.nanoTime() : 0;
+ openTransaction();
+ MTable mtable = ensureGetMTable(dbName, tblName);
+ if (doUseDirectSql) {
+ try {
+ Table table = convertToTable(mtable);
+ results = directSql.getPartitionsViaSqlFilter(table, dbName, tblName, parser);
+ } catch (Exception ex) {
+ LOG.error("Direct SQL failed, falling back to ORM", ex);
+ doUseDirectSql = false;
+ rollbackTransaction();
+ start = doTrace ? System.nanoTime() : 0;
+ openTransaction();
+ mtable = ensureGetMTable(dbName, tblName); // Detached on rollback, get again.
+ }
+ }
+ if (!doUseDirectSql) {
+ results = convertToParts(listMPartitionsByFilterNoTxn(
+ mtable, dbName, tblName, parser, maxParts));
+ }
+ success = commitTransaction();
+ LOG.info(results.size() + " partitions retrieved using " + (doUseDirectSql ? "SQL" : "ORM")
+ + (doTrace ? (" in " + ((System.nanoTime() - start) / 1000000.0) + "ms") : ""));
+ return results;
+ } finally {
+ if (!success) {
+ rollbackTransaction();
+ }
+ }
+ }
+
+ private MTable ensureGetMTable(String dbName, String tblName) throws NoSuchObjectException {
+ MTable mtable = getMTable(dbName, tblName);
+ if (mtable == null) {
+ throw new NoSuchObjectException("Specified database/table does not exist : "
+ + dbName + "." + tblName);
+ }
+ return mtable;
}
private FilterParser getFilterParser(String filter) throws MetaException {
@@ -1736,6 +1817,18 @@ public class ObjectStore implements RawS
* if mtable is null, generates the query to filter over tables in a database
*/
private String makeQueryFilterString(MTable mtable, String filter,
+ Map<String, Object> params) throws MetaException {
+ FilterParser parser =
+ (filter != null && filter.length() != 0) ? getFilterParser(filter) : null;
+ return makeQueryFilterString(mtable, parser, params);
+ }
+
+ /**
+ * Makes a JDO query filter string
+ * if mtable is not null, generates the query to filter over partitions in a table.
+ * if mtable is null, generates the query to filter over tables in a database
+ */
+ private String makeQueryFilterString(MTable mtable, FilterParser parser,
Map<String, Object> params)
throws MetaException {
@@ -1746,10 +1839,8 @@ public class ObjectStore implements RawS
queryBuilder.append("database.name == dbName");
}
- if (filter != null && filter.length() > 0) {
- FilterParser parser = getFilterParser(filter);
+ if (parser != null) {
String jdoFilter;
-
if (mtable != null) {
Table table = convertToTable(mtable);
jdoFilter = parser.tree.generateJDOFilter(table, params);
@@ -1794,54 +1885,35 @@ public class ObjectStore implements RawS
return paramDecl.toString();
}
- private List<MPartition> listMPartitionsByFilter(String dbName, String tableName,
- String filter, short maxParts) throws MetaException, NoSuchObjectException{
- boolean success = false;
+ private List<MPartition> listMPartitionsByFilterNoTxn(MTable mtable, String dbName,
+ String tableName, FilterParser parser, short maxParts)
+ throws MetaException, NoSuchObjectException {
List<MPartition> mparts = null;
- try {
- openTransaction();
- LOG.debug("Executing listMPartitionsByFilter");
- dbName = dbName.toLowerCase();
- tableName = tableName.toLowerCase();
+ LOG.debug("Executing listMPartitionsByFilterNoTxn");
+ Map<String, Object> params = new HashMap<String, Object>();
+ String queryFilterString = makeQueryFilterString(mtable, parser, params);
- MTable mtable = getMTable(dbName, tableName);
- if( mtable == null ) {
- throw new NoSuchObjectException("Specified database/table does not exist : "
- + dbName + "." + tableName);
- }
- Map<String, Object> params = new HashMap<String, Object>();
- String queryFilterString =
- makeQueryFilterString(mtable, filter, params);
-
- Query query = pm.newQuery(MPartition.class,
- queryFilterString);
+ Query query = pm.newQuery(MPartition.class,
+ queryFilterString);
- if( maxParts >= 0 ) {
- //User specified a row limit, set it on the Query
- query.setRange(0, maxParts);
- }
+ if( maxParts >= 0 ) {
+ //User specified a row limit, set it on the Query
+ query.setRange(0, maxParts);
+ }
- LOG.debug("Filter specified is " + filter + "," +
- " JDOQL filter is " + queryFilterString);
+ LOG.debug("JDOQL filter is " + queryFilterString);
- params.put("t1", tableName.trim());
- params.put("t2", dbName.trim());
+ params.put("t1", tableName.trim());
+ params.put("t2", dbName.trim());
- String parameterDeclaration = makeParameterDeclarationStringObj(params);
- query.declareParameters(parameterDeclaration);
- query.setOrdering("partitionName ascending");
+ String parameterDeclaration = makeParameterDeclarationStringObj(params);
+ query.declareParameters(parameterDeclaration);
+ query.setOrdering("partitionName ascending");
- mparts = (List<MPartition>) query.executeWithMap(params);
+ mparts = (List<MPartition>) query.executeWithMap(params);
- LOG.debug("Done executing query for listMPartitionsByFilter");
- pm.retrieveAll(mparts);
- success = commitTransaction();
- LOG.debug("Done retrieving all objects for listMPartitionsByFilter");
- } finally {
- if (!success) {
- rollbackTransaction();
- }
- }
+ LOG.debug("Done executing query for listMPartitionsByFilterNoTxn");
+ pm.retrieveAll(mparts);
return mparts;
}
@@ -1909,8 +1981,7 @@ public class ObjectStore implements RawS
return partNames;
}
Map<String, Object> params = new HashMap<String, Object>();
- String queryFilterString =
- makeQueryFilterString(mtable, filter, params);
+ String queryFilterString = makeQueryFilterString(mtable, filter, params);
Query query = pm.newQuery(
"select partitionName from org.apache.hadoop.hive.metastore.model.MPartition "
+ "where " + queryFilterString);
Modified: hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java?rev=1514554&r1=1514553&r2=1514554&view=diff
==============================================================================
--- hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java (original)
+++ hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java Fri Aug 16 01:21:54 2013
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.metastore;
+import java.io.IOException;
import java.lang.reflect.InvocationHandler;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
@@ -28,26 +29,36 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.thrift.TApplicationException;
import org.apache.thrift.TException;
import org.apache.thrift.protocol.TProtocolException;
import org.apache.thrift.transport.TTransportException;
+/**
+ * RetryingMetaStoreClient. Creates a proxy for a IMetaStoreClient
+ * implementation and retries calls to it on failure.
+ * If the login user is authenticated using keytab, it relogins user before
+ * each call.
+ *
+ */
public class RetryingMetaStoreClient implements InvocationHandler {
private static final Log LOG = LogFactory.getLog(RetryingMetaStoreClient.class.getName());
private final IMetaStoreClient base;
- private final HiveConf hiveConf;
private final int retryLimit;
private final int retryDelaySeconds;
+
+
protected RetryingMetaStoreClient(HiveConf hiveConf, HiveMetaHookLoader hookLoader,
Class<? extends IMetaStoreClient> msClientClass) throws MetaException {
- this.hiveConf = hiveConf;
this.retryLimit = hiveConf.getIntVar(HiveConf.ConfVars.METASTORETHRIFTFAILURERETRIES);
this.retryDelaySeconds =
hiveConf.getIntVar(HiveConf.ConfVars.METASTORE_CLIENT_CONNECT_RETRY_DELAY);
+
+ reloginExpiringKeytabUser();
this.base = (IMetaStoreClient) MetaStoreUtils.newInstance(msClientClass, new Class[] {
HiveConf.class, HiveMetaHookLoader.class}, new Object[] {hiveConf, hookLoader});
}
@@ -71,6 +82,10 @@ public class RetryingMetaStoreClient imp
TException caughtException = null;
while (true) {
try {
+ reloginExpiringKeytabUser();
+ if(retriesMade > 0){
+ base.reconnect();
+ }
ret = method.invoke(base, args);
break;
} catch (UndeclaredThrowableException e) {
@@ -95,8 +110,27 @@ public class RetryingMetaStoreClient imp
LOG.warn("MetaStoreClient lost connection. Attempting to reconnect.",
caughtException);
Thread.sleep(retryDelaySeconds * 1000);
- base.reconnect();
}
return ret;
}
+
+ /**
+ * Relogin if login user is logged in using keytab
+ * Relogin is actually done by ugi code only if sufficient time has passed
+ * A no-op if kerberos security is not enabled
+ * @throws MetaException
+ */
+ private void reloginExpiringKeytabUser() throws MetaException {
+ if(!ShimLoader.getHadoopShims().isSecurityEnabled()){
+ return;
+ }
+ try {
+ ShimLoader.getHadoopShims().reLoginUserFromKeytab();
+ } catch (IOException e) {
+ String msg = "Error doing relogin using keytab " + e.getMessage();
+ LOG.error(msg, e);
+ throw new MetaException(msg);
+ }
+ }
+
}