You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by am...@apache.org on 2013/04/01 09:01:00 UTC
svn commit: r1463091 [2/16] - in /hive/branches/HIVE-4115: ./ bin/ bin/ext/
common/src/gen/ common/src/java/org/apache/hadoop/hive/conf/ conf/
contrib/src/java/org/apache/hadoop/hive/contrib/serde2/
contrib/src/java/org/apache/hadoop/hive/contrib/serde...
Propchange: hive/branches/HIVE-4115/
------------------------------------------------------------------------------
svn:mergeinfo = /hive/trunk:1456340-1463086
Modified: hive/branches/HIVE-4115/.gitignore
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/.gitignore?rev=1463091&r1=1463090&r2=1463091&view=diff
==============================================================================
--- hive/branches/HIVE-4115/.gitignore (original)
+++ hive/branches/HIVE-4115/.gitignore Mon Apr 1 07:00:00 2013
@@ -8,3 +8,4 @@ build-eclipse
*.launch
*~
metastore_db
+common/src/gen
Modified: hive/branches/HIVE-4115/bin/hive
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/bin/hive?rev=1463091&r1=1463090&r2=1463091&view=diff
==============================================================================
--- hive/branches/HIVE-4115/bin/hive (original)
+++ hive/branches/HIVE-4115/bin/hive Mon Apr 1 07:00:00 2013
@@ -38,6 +38,10 @@ while [ $# -gt 0 ]; do
SERVICE=rcfilecat
shift
;;
+ --orcfiledump)
+ SERVICE=orcfiledump
+ shift
+ ;;
--help)
HELP=_help
shift
Modified: hive/branches/HIVE-4115/build-common.xml
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/build-common.xml?rev=1463091&r1=1463090&r2=1463091&view=diff
==============================================================================
--- hive/branches/HIVE-4115/build-common.xml (original)
+++ hive/branches/HIVE-4115/build-common.xml Mon Apr 1 07:00:00 2013
@@ -86,6 +86,7 @@
<pathelement location="${build.dir.hive}/hwi/test/classes"/>
<pathelement location="${build.dir.hive}/jdbc/test/classes"/>
<pathelement location="${build.dir.hive}/metastore/test/classes"/>
+ <pathelement location="${build.dir.hive}/hcatalog/test/classes"/>
<pathelement location="${build.dir.hive}/ql/test/classes"/>
<pathelement location="${build.dir.hive}/serde/test/classes"/>
<pathelement location="${build.dir.hive}/service/test/classes"/>
@@ -197,6 +198,7 @@
<pathelement location="${build.dir.hive}/common/classes"/>
<pathelement location="${build.dir.hive}/serde/classes"/>
<pathelement location="${build.dir.hive}/metastore/classes"/>
+ <pathelement location="${build.dir.hive}/hcatalog/classes"/>
<pathelement location="${build.dir.hive}/ql/classes"/>
<pathelement location="${build.dir.hive}/beeline/classes"/>
<pathelement location="${build.dir.hive}/cli/classes"/>
@@ -461,14 +463,14 @@
<sysproperty key="java.net.preferIPv4Stack" value="${java.net.preferIPv4Stack}"/>
<sysproperty key="hadoop.bin.path" value="${test.hadoop.bin.path}${junit.script.extension}"/>
<sysproperty key="test.concurrency.num.threads" value="${test.concurrency.num.threads}"/>
- <jvmarg line="${jvm.args}"/>
+ <jvmarg line="${junit.jvm.args}"/>
<classpath refid="test.local.classpath"/>
<formatter type="${test.junit.output.format}" usefile="${test.junit.output.usefile}" />
<batchtest todir="${test.build.dir}" unless="testcase">
<fileset dir="${test.build.classes}"
includes="**/${test.include}.class"
- excludes="**/TestSerDe.class,**/TestHiveMetaStore.class,**/*$*.class,${test.junit.exclude}" />
+ excludes="**/TestSerDe.class,**/TestHiveMetaStore.class,**/TestBeeLineDriver.class,**/TestHiveServer2Concurrency.class,**/*$*.class,${test.junit.exclude}" />
</batchtest>
<batchtest todir="${test.build.dir}" if="testcase">
<fileset dir="${test.build.classes}" includes="**/${testcase}.class"/>
Modified: hive/branches/HIVE-4115/build.properties
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/build.properties?rev=1463091&r1=1463090&r2=1463091&view=diff
==============================================================================
--- hive/branches/HIVE-4115/build.properties (original)
+++ hive/branches/HIVE-4115/build.properties Mon Apr 1 07:00:00 2013
@@ -72,10 +72,10 @@ jsp.test.jar=${hadoop.root}/lib/jetty-ex
common.jar=${hadoop.root}/lib/commons-httpclient-3.0.1.jar
# module names needed for build process
-iterate.hive.all=ant,shims,common,serde,metastore,ql,contrib,service,cli,jdbc,beeline,hwi,hbase-handler,pdk,builtins,testutils
-iterate.hive.modules=shims,common,serde,metastore,ql,contrib,service,cli,jdbc,beeline,hwi,hbase-handler,pdk,builtins,testutils
-iterate.hive.tests=ql,contrib,hbase-handler,hwi,jdbc,metastore,odbc,serde,service
-iterate.hive.thrift=ql,service,metastore,serde
+iterate.hive.all=ant,shims,common,serde,metastore,hcatalog,ql,contrib,service,cli,jdbc,beeline,hwi,hbase-handler,pdk,builtins,testutils
+iterate.hive.modules=shims,common,serde,metastore,hcatalog,ql,contrib,service,cli,jdbc,beeline,hwi,hbase-handler,pdk,builtins,testutils
+iterate.hive.tests=ql,contrib,hbase-handler,hwi,jdbc,metastore,hcatalog,odbc,serde,service
+iterate.hive.thrift=ql,service,metastore,hcatalog,serde
iterate.hive.protobuf=ql
iterate.hive.cpp=odbc
@@ -92,7 +92,8 @@ test.junit.timeout=43200000
# Use this property to selectively disable tests from the command line:
# ant test -Dtest.junit.exclude="**/TestCliDriver.class"
# ant test -Dtest.junit.exclude="**/Test*CliDriver.class,**/TestPartitions.class"
-test.junit.exclude="**/TestBeeLineDriver.class, **/TestHiveServer2Concurrency.class"
+test.junit.exclude=
+
test.continue.on.failure=false
test.submodule.exclude=
@@ -140,6 +141,9 @@ datanucleus.repo=http://www.datanucleus.
# JVM arguments
jvm.args=-XX:-UseSplitVerifier
+# junit jvm args
+junit.jvm.args=-XX:-UseSplitVerifier -XX:+CMSClassUnloadingEnabled -XX:MaxPermSize=128M
+
#
# Eclipse Properties
#
Modified: hive/branches/HIVE-4115/build.xml
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/build.xml?rev=1463091&r1=1463090&r2=1463091&view=diff
==============================================================================
--- hive/branches/HIVE-4115/build.xml (original)
+++ hive/branches/HIVE-4115/build.xml Mon Apr 1 07:00:00 2013
@@ -136,6 +136,7 @@
<pathelement location="${build.dir.hive}/common/classes"/>
<pathelement location="${build.dir.hive}/serde/classes"/>
<pathelement location="${build.dir.hive}/metastore/classes"/>
+ <pathelement location="${build.dir.hive}/hcatalog/classes"/>
<pathelement location="${build.dir.hive}/ql/classes"/>
<pathelement location="${build.dir.hive}/cli/classes"/>
<pathelement location="${build.dir.hive}/beeline/classes"/>
@@ -509,6 +510,14 @@
<fileset dir="${target.bin.dir}"/>
</chmod>
+ <!-- Package the hcat stuff and pull it up into Hive's build dir -->
+ <ant antfile="${hive.root}/hcatalog/build.xml" target="package"
+ inheritAll="false"/>
+ <mkdir dir="${build.dir.hive}/hcatalog"/>
+ <copy todir="${build.dir.hive}/hcatalog">
+ <fileset dir="${hive.root}/hcatalog/build/hcatalog-${version}"/>
+ </copy>
+
<!-- special case because builtins compilation depends on packaging
up everything else first -->
<ant antfile="${hive.root}/builtins/build.xml" target="package"
@@ -666,6 +675,7 @@
<packageset dir="metastore/src/test"/>
<packageset dir="metastore/src/gen/thrift/gen-javabean"/>
<packageset dir="metastore/src/model"/>
+ <packageset dir="hcatalog/src/java"/>
<packageset dir="cli/src/java"/>
<packageset dir="beeline/src/java"/>
<packageset dir="ql/src/java"/>
@@ -756,6 +766,9 @@
<exclude name="docs/**"/>
<exclude name="lib/py/**/*-remote"/>
</tarfileset>
+ <tarfileset dir="${build.dir.hive}/hcatalog" mode="755" prefix="${bin.final.name}"
+ excludes="${vcs.excludes}">
+ </tarfileset>
</param.listofitems>
</macro_tar>
</target>
@@ -949,6 +962,7 @@
<sourcePath path="${hive.root}/hwi" />
<sourcePath path="${hive.root}/jdbc" />
<sourcePath path="${hive.root}/metastore" />
+ <sourcePath path="${hive.root}/hcatalog" />
<sourcePath path="${hive.root}/odbc" />
<sourcePath path="${hive.root}/pdk" />
<sourcePath path="${hive.root}/ql" />
@@ -965,6 +979,7 @@
<class location="${build.dir.hive}/hwi/classes" />
<class location="${build.dir.hive}/jdbc/classes" />
<class location="${build.dir.hive}/metastore/classes" />
+ <class location="${build.dir.hive}/hcatalog/classes" />
<class location="${build.dir.hive}/pdk/classes" />
<class location="${build.dir.hive}/ql/classes" />
<class location="${build.dir.hive}/serde/classes" />
@@ -1066,6 +1081,8 @@
todir="${mvn.jar.dir}" />
<copy file="${build.dir.hive}/metastore/hive-metastore-${version}.jar"
todir="${mvn.jar.dir}" />
+ <copy file="${build.dir.hive}/hcatalog/hive-hcatalog-${version}.jar"
+ todir="${mvn.jar.dir}" />
<copy file="${build.dir.hive}/pdk/hive-pdk-${version}.jar"
todir="${mvn.jar.dir}" />
<copy file="${build.dir.hive}/ql/hive-exec-${version}.jar"
@@ -1098,6 +1115,8 @@
tofile="${mvn.pom.dir}/hive-jdbc-${version}.pom" />
<copy file="${build.dir.hive}/metastore/pom.xml"
tofile="${mvn.pom.dir}/hive-metastore-${version}.pom" />
+ <copy file="${build.dir.hive}/hcatalog/pom.xml"
+ tofile="${mvn.pom.dir}/hive-hcatalog-${version}.pom" />
<copy file="${build.dir.hive}/pdk/pom.xml"
tofile="${mvn.pom.dir}/hive-pdk-${version}.pom" />
<copy file="${build.dir.hive}/ql/pom.xml"
@@ -1215,6 +1234,9 @@
<param name="hive.project" value="metastore" />
</antcall>
<antcall target="maven-publish-artifact">
+ <param name="hive.project" value="hcatalog" />
+ </antcall>
+ <antcall target="maven-publish-artifact">
<param name="hive.project" value="pdk" />
</antcall>
<antcall target="maven-publish-artifact">
@@ -1250,6 +1272,8 @@
</exec>
</sequential>
</macrodef>
+
+ <!-- hive-anttasks -->
<sign-artifact
input.file="${mvn.jar.dir}/hive-anttasks-${version}.jar"
output.file="${mvn.jar.dir}/hive-anttasks-${version}.jar.asc"
@@ -1258,6 +1282,8 @@
input.file="${mvn.pom.dir}/hive-anttasks-${version}.pom"
output.file="${mvn.pom.dir}/hive-anttasks-${version}.pom.asc"
gpg.passphrase="${gpg.passphrase}"/>
+
+ <!-- hive-beeline -->
<sign-artifact
input.file="${mvn.jar.dir}/hive-beeline-${version}.jar"
output.file="${mvn.jar.dir}/hive-beeline-${version}.jar.asc"
@@ -1266,6 +1292,8 @@
input.file="${mvn.pom.dir}/hive-beeline-${version}.pom"
output.file="${mvn.pom.dir}/hive-beeline-${version}.pom.asc"
gpg.passphrase="${gpg.passphrase}"/>
+
+ <!-- hive-cli -->
<sign-artifact
input.file="${mvn.jar.dir}/hive-cli-${version}.jar"
output.file="${mvn.jar.dir}/hive-cli-${version}.jar.asc"
@@ -1274,6 +1302,8 @@
input.file="${mvn.pom.dir}/hive-cli-${version}.pom"
output.file="${mvn.pom.dir}/hive-cli-${version}.pom.asc"
gpg.passphrase="${gpg.passphrase}"/>
+
+ <!-- hive-common -->
<sign-artifact
input.file="${mvn.jar.dir}/hive-common-${version}.jar"
output.file="${mvn.jar.dir}/hive-common-${version}.jar.asc"
@@ -1282,6 +1312,8 @@
input.file="${mvn.pom.dir}/hive-common-${version}.pom"
output.file="${mvn.pom.dir}/hive-common-${version}.pom.asc"
gpg.passphrase="${gpg.passphrase}"/>
+
+ <!-- hive-contrib -->
<sign-artifact
input.file="${mvn.jar.dir}/hive-contrib-${version}.jar"
output.file="${mvn.jar.dir}/hive-contrib-${version}.jar.asc"
@@ -1290,6 +1322,8 @@
input.file="${mvn.pom.dir}/hive-contrib-${version}.pom"
output.file="${mvn.pom.dir}/hive-contrib-${version}.pom.asc"
gpg.passphrase="${gpg.passphrase}"/>
+
+ <!-- hive-exec -->
<sign-artifact
input.file="${mvn.jar.dir}/hive-exec-${version}.jar"
output.file="${mvn.jar.dir}/hive-exec-${version}.jar.asc"
@@ -1298,6 +1332,8 @@
input.file="${mvn.pom.dir}/hive-exec-${version}.pom"
output.file="${mvn.pom.dir}/hive-exec-${version}.pom.asc"
gpg.passphrase="${gpg.passphrase}"/>
+
+ <!-- hive-hbase-handler -->
<sign-artifact
input.file="${mvn.jar.dir}/hive-hbase-handler-${version}.jar"
output.file="${mvn.jar.dir}/hive-hbase-handler-${version}.jar.asc"
@@ -1306,6 +1342,8 @@
input.file="${mvn.pom.dir}/hive-hbase-handler-${version}.pom"
output.file="${mvn.pom.dir}/hive-hbase-handler-${version}.pom.asc"
gpg.passphrase="${gpg.passphrase}"/>
+
+ <!-- hive-hwi -->
<sign-artifact
input.file="${mvn.jar.dir}/hive-hwi-${version}.jar"
output.file="${mvn.jar.dir}/hive-hwi-${version}.jar.asc"
@@ -1314,6 +1352,8 @@
input.file="${mvn.pom.dir}/hive-hwi-${version}.pom"
output.file="${mvn.pom.dir}/hive-hwi-${version}.pom.asc"
gpg.passphrase="${gpg.passphrase}"/>
+
+ <!-- hive-jdbc -->
<sign-artifact
input.file="${mvn.jar.dir}/hive-jdbc-${version}.jar"
output.file="${mvn.jar.dir}/hive-jdbc-${version}.jar.asc"
@@ -1322,22 +1362,38 @@
input.file="${mvn.pom.dir}/hive-jdbc-${version}.pom"
output.file="${mvn.pom.dir}/hive-jdbc-${version}.pom.asc"
gpg.passphrase="${gpg.passphrase}"/>
+
+ <!-- hive-metastore -->
<sign-artifact
input.file="${mvn.jar.dir}/hive-metastore-${version}.jar"
output.file="${mvn.jar.dir}/hive-metastore-${version}.jar.asc"
gpg.passphrase="${gpg.passphrase}"/>
<sign-artifact
- input.file="${mvn.pom.dir}/hive-metastore-${version}.pom"
- output.file="${mvn.pom.dir}/hive-metastore-${version}.pom.asc"
+ input.file="${mvn.jar.dir}/hive-metastore-${version}.pom"
+ output.file="${mvn.jar.dir}/hive-metastore-${version}.pom.asc"
gpg.passphrase="${gpg.passphrase}"/>
+
+ <!-- hive-hcatalog -->
<sign-artifact
- input.file="${mvn.jar.dir}/hive-pdk-${version}.jar"
- output.file="${mvn.jar.dir}/hive-pdk-${version}.jar.asc"
+ input.file="${mvn.pom.dir}/hive-hcatalog-${version}.jar"
+ output.file="${mvn.pom.dir}/hive-hcatalog-${version}.jar.asc"
+ gpg.passphrase="${gpg.passphrase}"/>
+ <sign-artifact
+ input.file="${mvn.pom.dir}/hive-hcatalog-${version}.pom"
+ output.file="${mvn.pom.dir}/hive-hcatalog-${version}.pom.asc"
+ gpg.passphrase="${gpg.passphrase}"/>
+
+ <!-- hive-pdk -->
+ <sign-artifact
+ input.file="${mvn.pom.dir}/hive-pdk-${version}.jar"
+ output.file="${mvn.pom.dir}/hive-pdk-${version}.jar.asc"
gpg.passphrase="${gpg.passphrase}"/>
<sign-artifact
input.file="${mvn.pom.dir}/hive-pdk-${version}.pom"
output.file="${mvn.pom.dir}/hive-pdk-${version}.pom.asc"
gpg.passphrase="${gpg.passphrase}"/>
+
+ <!-- hive-serde -->
<sign-artifact
input.file="${mvn.jar.dir}/hive-serde-${version}.jar"
output.file="${mvn.jar.dir}/hive-serde-${version}.jar.asc"
@@ -1346,6 +1402,8 @@
input.file="${mvn.pom.dir}/hive-serde-${version}.pom"
output.file="${mvn.pom.dir}/hive-serde-${version}.pom.asc"
gpg.passphrase="${gpg.passphrase}"/>
+
+ <!-- hive-service -->
<sign-artifact
input.file="${mvn.jar.dir}/hive-service-${version}.jar"
output.file="${mvn.jar.dir}/hive-service-${version}.jar.asc"
@@ -1354,6 +1412,8 @@
input.file="${mvn.pom.dir}/hive-service-${version}.pom"
output.file="${mvn.pom.dir}/hive-service-${version}.pom.asc"
gpg.passphrase="${gpg.passphrase}"/>
+
+ <!-- hive-shims -->
<sign-artifact
input.file="${mvn.jar.dir}/hive-shims-${version}.jar"
output.file="${mvn.jar.dir}/hive-shims-${version}.jar.asc"
Modified: hive/branches/HIVE-4115/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1463091&r1=1463090&r2=1463091&view=diff
==============================================================================
--- hive/branches/HIVE-4115/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/branches/HIVE-4115/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Mon Apr 1 07:00:00 2013
@@ -223,6 +223,9 @@ public class HiveConf extends Configurat
// not specified
DROPIGNORESNONEXISTENT("hive.exec.drop.ignorenonexistent", true),
+ // ignore the mapjoin hint
+ HIVEIGNOREMAPJOINHINT("hive.ignore.mapjoin.hint", true),
+
// Hadoop Configuration Properties
// Properties with null values are ignored and exist only for the purpose of giving us
// a symbolic name to reference in the Hive source code. Properties with non-null
@@ -484,7 +487,7 @@ public class HiveConf extends Configurat
HIVESKEWJOIN("hive.optimize.skewjoin", false),
HIVECONVERTJOIN("hive.auto.convert.join", true),
- HIVECONVERTJOINNOCONDITIONALTASK("hive.auto.convert.join.noconditionaltask", false),
+ HIVECONVERTJOINNOCONDITIONALTASK("hive.auto.convert.join.noconditionaltask", true),
HIVECONVERTJOINNOCONDITIONALTASKTHRESHOLD("hive.auto.convert.join.noconditionaltask.size",
10000000L),
HIVESKEWJOINKEY("hive.skewjoin.key", 100000),
@@ -603,6 +606,7 @@ public class HiveConf extends Configurat
// higher compute cost.
HIVE_STATS_NDV_ERROR("hive.stats.ndv.error", (float)20.0),
HIVE_STATS_KEY_PREFIX_MAX_LENGTH("hive.stats.key.prefix.max.length", 200),
+ HIVE_STATS_KEY_PREFIX("hive.stats.key.prefix", ""), // internal usage only
// Concurrency
HIVE_SUPPORT_CONCURRENCY("hive.support.concurrency", false),
Modified: hive/branches/HIVE-4115/conf/hive-default.xml.template
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/conf/hive-default.xml.template?rev=1463091&r1=1463090&r2=1463091&view=diff
==============================================================================
--- hive/branches/HIVE-4115/conf/hive-default.xml.template (original)
+++ hive/branches/HIVE-4115/conf/hive-default.xml.template Mon Apr 1 07:00:00 2013
@@ -785,6 +785,12 @@
</property>
<property>
+ <name>hive.ignore.mapjoin.hint</name>
+ <value>true</value>
+ <description>Ignore the mapjoin hint</description>
+</property>
+
+<property>
<name>hive.mapjoin.localtask.max.memory.usage</name>
<value>0.90</value>
<description>This number means how much memory the local task can take to hold the key/value into in-memory hash table; If the local task's memory usage is more than this number, the local task will be abort by themself. It means the data of small table is too large to be hold in the memory.</description>
@@ -810,7 +816,7 @@
<property>
<name>hive.auto.convert.join.noconditionaltask</name>
- <value>false</value>
+ <value>true</value>
<description>Whether Hive enable the optimization about converting common join into mapjoin based on the input file
size. If this paramater is on, and the sum of size for n-1 of the tables/partitions for a n-way join is smaller than the
specified size, the join is directly converted to a mapjoin (there is no conditional task).
Modified: hive/branches/HIVE-4115/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/RegexSerDe.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/RegexSerDe.java?rev=1463091&r1=1463090&r2=1463091&view=diff
==============================================================================
--- hive/branches/HIVE-4115/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/RegexSerDe.java (original)
+++ hive/branches/HIVE-4115/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/RegexSerDe.java Mon Apr 1 07:00:00 2013
@@ -29,7 +29,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.serde.serdeConstants;
-import org.apache.hadoop.hive.serde2.SerDe;
+import org.apache.hadoop.hive.serde2.AbstractSerDe;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.SerDeStats;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -70,7 +70,7 @@ import org.apache.hadoop.io.Writable;
* writableStringObjectInspector. We should switch to that when we have a UTF-8
* based Regex library.
*/
-public class RegexSerDe implements SerDe {
+public class RegexSerDe extends AbstractSerDe {
public static final Log LOG = LogFactory.getLog(RegexSerDe.class.getName());
Modified: hive/branches/HIVE-4115/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/TypedBytesSerDe.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/TypedBytesSerDe.java?rev=1463091&r1=1463090&r2=1463091&view=diff
==============================================================================
--- hive/branches/HIVE-4115/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/TypedBytesSerDe.java (original)
+++ hive/branches/HIVE-4115/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/TypedBytesSerDe.java Mon Apr 1 07:00:00 2013
@@ -32,7 +32,7 @@ import org.apache.hadoop.hive.contrib.ut
import org.apache.hadoop.hive.ql.io.NonSyncDataInputBuffer;
import org.apache.hadoop.hive.ql.io.NonSyncDataOutputBuffer;
import org.apache.hadoop.hive.serde.serdeConstants;
-import org.apache.hadoop.hive.serde2.SerDe;
+import org.apache.hadoop.hive.serde2.AbstractSerDe;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.SerDeStats;
import org.apache.hadoop.hive.serde2.SerDeUtils;
@@ -40,11 +40,11 @@ import org.apache.hadoop.hive.serde2.io.
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.io.ShortWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector;
@@ -73,7 +73,7 @@ import org.apache.hadoop.io.Writable;
* this, which is apparently 25% faster than the python version is available at
* http://github.com/klbostee/ctypedbytes/tree/master
*/
-public class TypedBytesSerDe implements SerDe {
+public class TypedBytesSerDe extends AbstractSerDe {
public static final Log LOG = LogFactory.getLog(TypedBytesSerDe.class
.getName());
Modified: hive/branches/HIVE-4115/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/s3/S3LogDeserializer.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/s3/S3LogDeserializer.java?rev=1463091&r1=1463090&r2=1463091&view=diff
==============================================================================
--- hive/branches/HIVE-4115/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/s3/S3LogDeserializer.java (original)
+++ hive/branches/HIVE-4115/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/s3/S3LogDeserializer.java Mon Apr 1 07:00:00 2013
@@ -27,7 +27,7 @@ import java.util.regex.Pattern;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.serde2.Deserializer;
+import org.apache.hadoop.hive.serde2.AbstractDeserializer;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.SerDeStats;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -42,7 +42,7 @@ import org.apache.hadoop.io.Writable;
* S3LogDeserializer.
*
*/
-public class S3LogDeserializer implements Deserializer {
+public class S3LogDeserializer extends AbstractDeserializer {
public static final Log LOG = LogFactory.getLog(S3LogDeserializer.class
.getName());
@@ -73,6 +73,7 @@ public class S3LogDeserializer implement
S3LogStruct deserializeCache = new S3LogStruct();
+ @Override
public void initialize(Configuration job, Properties tbl)
throws SerDeException {
@@ -132,6 +133,7 @@ public class S3LogDeserializer implement
return (c);
}
+ @Override
public Object deserialize(Writable field) throws SerDeException {
String row = null;
if (field instanceof BytesWritable) {
@@ -155,6 +157,7 @@ public class S3LogDeserializer implement
}
}
+ @Override
public ObjectInspector getObjectInspector() throws SerDeException {
return cachedObjectInspector;
}
@@ -202,6 +205,7 @@ public class S3LogDeserializer implement
}
+ @Override
public SerDeStats getSerDeStats() {
// no support for statistics
return null;
Modified: hive/branches/HIVE-4115/data/conf/hive-site.xml
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/data/conf/hive-site.xml?rev=1463091&r1=1463090&r2=1463091&view=diff
==============================================================================
--- hive/branches/HIVE-4115/data/conf/hive-site.xml (original)
+++ hive/branches/HIVE-4115/data/conf/hive-site.xml Mon Apr 1 07:00:00 2013
@@ -177,6 +177,12 @@
</property>
<property>
+ <name>hive.ignore.mapjoin.hint</name>
+ <value>false</value>
+ <description>Whether Hive ignores the mapjoin hint</description>
+</property>
+
+<property>
<name>hive.input.format</name>
<value>org.apache.hadoop.hive.ql.io.CombineHiveInputFormat</value>
<description>The default input format, if it is not specified, the system assigns it. It is set to HiveInputFormat for hadoop versions 17, 18 and 19, whereas it is set to CombineHiveInputFormat for hadoop 20. The user can always overwrite it - if there is a bug in CombineHiveInputFormat, it can always be manually set to HiveInputFormat. </description>
Modified: hive/branches/HIVE-4115/eclipse-templates/.classpath
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/eclipse-templates/.classpath?rev=1463091&r1=1463090&r2=1463091&view=diff
==============================================================================
--- hive/branches/HIVE-4115/eclipse-templates/.classpath (original)
+++ hive/branches/HIVE-4115/eclipse-templates/.classpath Mon Apr 1 07:00:00 2013
@@ -40,12 +40,12 @@
<classpathentry kind="lib" path="build/ivy/lib/hadoop0.20.shim/jetty-util-6.1.14.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/hadoop0.20.shim/jsp-2.1-6.1.14.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/hadoop0.20.shim/jsp-api-2.1-6.1.14.jar"/>
- <classpathentry kind="lib" path="build/ivy/lib/hadoop0.20.shim/junit-4.5.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/hadoop0.20.shim/kfs-0.3.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/hadoop0.20.shim/mina-core-2.0.0-M5.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/hadoop0.20.shim/oro-2.0.8.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/hadoop0.20.shim/servlet-api-2.5-6.1.14.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/hadoop0.20.shim/xmlenc-0.52.jar"/>
+ <classpathentry kind="lib" path="build/ivy/lib/hadoop0.20.shim/guava-@guava-hadoop20.version@.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/default/avro-@avro.version@.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/default/avro-mapred-@avro.version@.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/default/jline-@jline.version@.jar"/>
@@ -56,8 +56,8 @@
<classpathentry kind="lib" path="build/ivy/lib/default/commons-lang-@commons-lang.version@.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/default/commons-logging-@commons-logging.version@.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/default/commons-logging-api-@commons-logging-api.version@.jar"/>
+ <classpathentry kind="lib" path="build/ivy/lib/default/commons-io-@commons-io.version@.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/default/derby-@derby.version@.jar"/>
- <classpathentry kind="lib" path="build/ivy/lib/default/guava-@guava-hadoop20.version@.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/default/hbase-@hbase.version@.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/default/hbase-@hbase.version@-tests.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/default/libfb303-@libfb303.version@.jar"/>
@@ -86,22 +86,38 @@
<classpathentry kind="lib" path="build/ivy/lib/default/jackson-mapper-asl-1.8.8.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/default/jackson-xc-1.8.8.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/default/mockito-all-@mockito-all.version@.jar"/>
- <classpathentry kind="lib" path="build/ivy/lib/default/stringtemplate-@stringtemplate.version@.jar"/>
+ <classpathentry kind="lib" path="build/ivy/lib/default/ST4-@ST4.version@.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/default/snappy-@snappy.version@.jar"/>
+ <classpathentry kind="lib" path="build/beeline/hive-beeline-@HIVE_VERSION@.jar"/>
+ <classpathentry kind="lib" path="build/ivy/lib/default/tempus-fugit-@tempus-fugit.version@.jar"/>
<classpathentry kind="lib" path="build/builtins/hive-builtins-@HIVE_VERSION@.jar"/>
<classpathentry kind="src" path="build/contrib/test/src"/>
<classpathentry kind="src" path="build/metastore/gen/antlr/gen-java"/>
+ <classpathentry kind="lib" path="build/testutils/hive-testutils-@HIVE_VERSION@.jar"/>
<classpathentry kind="src" path="build/ql/test/src"/>
<classpathentry kind="src" path="build/ql/gen/antlr/gen-java"/>
+ <classpathentry kind="src" path="beeline/src/java"/>
+ <classpathentry kind="src" path="beeline/src/test"/>
+ <classpathentry kind="src" path="builtins/src"/>
<classpathentry kind="src" path="cli/src/java"/>
+ <classpathentry kind="src" path="cli/src/test"/>
<classpathentry kind="src" path="common/src/java"/>
<classpathentry kind="src" path="common/src/test"/>
<classpathentry kind="src" path="contrib/src/java"/>
<classpathentry kind="src" path="contrib/src/test"/>
+ <classpathentry kind="src" path="hbase-handler/src/java"/>
+ <classpathentry kind="src" path="hwi/src/java"/>
+ <classpathentry kind="src" path="hwi/src/test"/>
+ <classpathentry kind="src" path="jdbc/src/java"/>
+ <classpathentry kind="src" path="jdbc/src/test"/>
+ <classpathentry kind="src" path="hcatalog/src/java"/>
+ <classpathentry kind="src" path="hcatalog/src/test"/>
<classpathentry kind="src" path="metastore/src/gen/thrift/gen-javabean"/>
<classpathentry kind="src" path="metastore/src/java"/>
<classpathentry kind="src" path="metastore/src/model"/>
<classpathentry kind="src" path="metastore/src/test"/>
+ <classpathentry kind="src" path="pdk/src/java"/>
+ <classpathentry kind="src" path="pdk/test-plugin/src"/>
<classpathentry kind="src" path="ql/src/gen/thrift/gen-javabean"/>
<classpathentry kind="src" path="ql/src/gen/protobuf/gen-java"/>
<classpathentry kind="src" path="ql/src/java"/>
@@ -113,17 +129,9 @@
<classpathentry kind="src" path="service/src/gen/thrift/gen-javabean"/>
<classpathentry kind="src" path="service/src/java"/>
<classpathentry kind="src" path="service/src/test"/>
- <classpathentry kind="src" path="jdbc/src/java"/>
- <classpathentry kind="src" path="jdbc/src/test"/>
<classpathentry kind="src" path="shims/src/@HADOOPVERPREF@/java"/>
<classpathentry kind="src" path="shims/src/common/java"/>
- <classpathentry kind="src" path="hwi/src/java"/>
- <classpathentry kind="src" path="hwi/src/test"/>
- <classpathentry kind="src" path="hbase-handler/src/java"/>
<classpathentry excluding="queries/|results/|templates/" kind="src" path="hbase-handler/src/test"/>
- <classpathentry kind="src" path="pdk/src/java"/>
- <classpathentry kind="src" path="pdk/test-plugin/src"/>
- <classpathentry kind="src" path="builtins/src"/>
- <classpathentry kind="src" path="cli/src/test"/>
+ <classpathentry kind="src" path="testutils/src/java"/>
<classpathentry kind="output" path="build/eclipse-classes"/>
</classpath>
Modified: hive/branches/HIVE-4115/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java?rev=1463091&r1=1463090&r2=1463091&view=diff
==============================================================================
--- hive/branches/HIVE-4115/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java (original)
+++ hive/branches/HIVE-4115/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java Mon Apr 1 07:00:00 2013
@@ -31,6 +31,7 @@ import org.apache.hadoop.hbase.client.Pu
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.AbstractSerDe;
import org.apache.hadoop.hive.serde2.ByteStream;
import org.apache.hadoop.hive.serde2.SerDe;
import org.apache.hadoop.hive.serde2.SerDeException;
@@ -58,7 +59,7 @@ import org.apache.hadoop.io.Writable;
* HBaseSerDe can be used to serialize object into an HBase table and
* deserialize objects from an HBase table.
*/
-public class HBaseSerDe implements SerDe {
+public class HBaseSerDe extends AbstractSerDe {
public static final String HBASE_COLUMNS_MAPPING = "hbase.columns.mapping";
public static final String HBASE_TABLE_NAME = "hbase.table.name";
@@ -546,10 +547,11 @@ public class HBaseSerDe implements SerDe
throw new SerDeException("HBase row key cannot be NULL");
}
- if(putTimestamp >= 0)
+ if(putTimestamp >= 0) {
put = new Put(key,putTimestamp);
- else
+ } else {
put = new Put(key);
+ }
// Serialize each field
for (int i = 0; i < fields.size(); i++) {
Propchange: hive/branches/HIVE-4115/hcatalog/historical/branches/branch-0.4/
------------------------------------------------------------------------------
--- svn:externals (added)
+++ svn:externals Mon Apr 1 07:00:00 2013
@@ -0,0 +1 @@
+src/test/e2e/harness http://svn.apache.org/repos/asf/pig/trunk/test/e2e/harness
Propchange: hive/branches/HIVE-4115/hcatalog/historical/branches/branch-0.4/
------------------------------------------------------------------------------
--- svn:mergeinfo (added)
+++ svn:mergeinfo Mon Apr 1 07:00:00 2013
@@ -0,0 +1,2 @@
+/hive/trunk/hcatalog/historical/branches/branch-0.4:1456340-1463086
+/incubator/hcatalog/trunk:1296568,1299080,1304536,1307029,1325864,1327167,1327285,1329321,1332736,1332807,1332858,1336456,1338996,1340149,1365770,1373743,1388158,1392116
Propchange: hive/branches/HIVE-4115/hcatalog/historical/branches/branch-0.4.0-rc4/
------------------------------------------------------------------------------
--- svn:externals (added)
+++ svn:externals Mon Apr 1 07:00:00 2013
@@ -0,0 +1 @@
+src/test/e2e/harness http://svn.apache.org/repos/asf/pig/trunk/test/e2e/harness
Propchange: hive/branches/HIVE-4115/hcatalog/historical/branches/branch-0.4.0-rc4/
------------------------------------------------------------------------------
--- svn:mergeinfo (added)
+++ svn:mergeinfo Mon Apr 1 07:00:00 2013
@@ -0,0 +1,2 @@
+/hive/trunk/hcatalog/historical/branches/branch-0.4.0-rc4:1456340-1463086
+/incubator/hcatalog/trunk:1296568,1299080,1304536,1307029
Propchange: hive/branches/HIVE-4115/hcatalog/historical/branches/branch-0.4.0-rc4/hive/
------------------------------------------------------------------------------
--- svn:externals (added)
+++ svn:externals Mon Apr 1 07:00:00 2013
@@ -0,0 +1 @@
+
Propchange: hive/branches/HIVE-4115/hcatalog/historical/branches/branch-0.4/hive/
------------------------------------------------------------------------------
--- svn:externals (added)
+++ svn:externals Mon Apr 1 07:00:00 2013
@@ -0,0 +1 @@
+
Propchange: hive/branches/HIVE-4115/hcatalog/historical/branches/branch-0.5/
------------------------------------------------------------------------------
--- svn:externals (added)
+++ svn:externals Mon Apr 1 07:00:00 2013
@@ -0,0 +1 @@
+src/test/e2e/harness http://svn.apache.org/repos/asf/pig/trunk/test/e2e/harness
Propchange: hive/branches/HIVE-4115/hcatalog/historical/site/author/src/documentation/
------------------------------------------------------------------------------
--- svn:externals (added)
+++ svn:externals Mon Apr 1 07:00:00 2013
@@ -0,0 +1 @@
+skins http://svn.apache.org/repos/asf/hadoop/common/site/main/author/src/documentation/skins
Propchange: hive/branches/HIVE-4115/hcatalog/historical/trunk/
------------------------------------------------------------------------------
--- svn:externals (added)
+++ svn:externals Mon Apr 1 07:00:00 2013
@@ -0,0 +1 @@
+src/test/e2e/harness http://svn.apache.org/repos/asf/pig/trunk/test/e2e/harness
Modified: hive/branches/HIVE-4115/ivy/libraries.properties
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ivy/libraries.properties?rev=1463091&r1=1463090&r2=1463091&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ivy/libraries.properties (original)
+++ hive/branches/HIVE-4115/ivy/libraries.properties Mon Apr 1 07:00:00 2013
@@ -65,9 +65,9 @@ sqlline.version=1_0_2
sqlline.branch=1.0.2
slf4j-api.version=1.6.1
slf4j-log4j12.version=1.6.1
+ST4.version=4.0.4
tempus-fugit.version=1.1
snappy.version=0.2
velocity.version=1.5
zookeeper.version=3.4.3
javolution.version=5.5.1
-stringtemplate.version=3.2.1
Modified: hive/branches/HIVE-4115/jdbc/src/test/org/apache/hive/jdbc/TestJdbcDriver2.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/jdbc/src/test/org/apache/hive/jdbc/TestJdbcDriver2.java?rev=1463091&r1=1463090&r2=1463091&view=diff
==============================================================================
--- hive/branches/HIVE-4115/jdbc/src/test/org/apache/hive/jdbc/TestJdbcDriver2.java (original)
+++ hive/branches/HIVE-4115/jdbc/src/test/org/apache/hive/jdbc/TestJdbcDriver2.java Mon Apr 1 07:00:00 2013
@@ -811,17 +811,19 @@ public class TestJdbcDriver2 extends Tes
ResultSet res = stmt.executeQuery("describe " + tableName);
+ res.next(); // skip header 1
+ res.next(); // skip header 2
+
res.next();
- assertEquals("Column name 'under_col' not found", "under_col", res.getString(1));
+ assertEquals("Column name 'under_col' not found", "under_col", res.getString(1).trim());
assertEquals("Column type 'under_col' for column under_col not found", "int", res
- .getString(2));
+ .getString(2).trim());
res.next();
- assertEquals("Column name 'value' not found", "value", res.getString(1));
+ assertEquals("Column name 'value' not found", "value", res.getString(1).trim());
assertEquals("Column type 'string' for column key not found", "string", res
- .getString(2));
+ .getString(2).trim());
assertFalse("More results found than expected", res.next());
-
}
public void testDatabaseMetaData() throws SQLException {
Modified: hive/branches/HIVE-4115/metastore/ivy.xml
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/metastore/ivy.xml?rev=1463091&r1=1463090&r2=1463091&view=diff
==============================================================================
--- hive/branches/HIVE-4115/metastore/ivy.xml (original)
+++ hive/branches/HIVE-4115/metastore/ivy.xml Mon Apr 1 07:00:00 2013
@@ -26,10 +26,11 @@
<include file="${ivy.conf.dir}/common-configurations.xml"/>
</configurations>
<dependencies>
+ <dependency org="org.antlr" name="antlr" rev="${antlr.version}" transitive="false"/>
+ <dependency org="org.antlr" name="antlr-runtime" rev="${antlr-runtime.version}" transitive="false"/>
+ <dependency org="org.antlr" name="ST4" rev="${ST4.version}" transitive="false"/><!-- manually added (antlr dep), bad POM -->
<dependency org="org.apache.hive" name="hive-serde" rev="${version}"
conf="compile->default" />
- <dependency org="org.antlr" name="antlr" rev="${antlr.version}"/>
- <dependency org="org.antlr" name="antlr-runtime" rev="${antlr-runtime.version}"/>
<dependency org="commons-dbcp" name="commons-dbcp" rev="${commons-dbcp.version}">
<exclude module="commons-pool" />
<exclude org="org.apache.geronimo.specs" module="geronimo-jta_1.1_spec"/>
Modified: hive/branches/HIVE-4115/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java?rev=1463091&r1=1463090&r2=1463091&view=diff
==============================================================================
--- hive/branches/HIVE-4115/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java (original)
+++ hive/branches/HIVE-4115/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java Mon Apr 1 07:00:00 2013
@@ -28,13 +28,10 @@ import org.apache.commons.lang.exception
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.common.classification.InterfaceAudience;
import org.apache.hadoop.hive.common.classification.InterfaceStability;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.hooks.JDOConnectionURLHook;
-import org.apache.hadoop.util.ReflectionUtils;
@InterfaceAudience.Private
@InterfaceStability.Evolving
@@ -43,7 +40,7 @@ public class RetryingHMSHandler implemen
private static final Log LOG = LogFactory.getLog(RetryingHMSHandler.class);
private final IHMSHandler base;
- private MetaStoreInit.MetaStoreInitData metaStoreInitData =
+ private final MetaStoreInit.MetaStoreInitData metaStoreInitData =
new MetaStoreInit.MetaStoreInitData();
private final HiveConf hiveConf;
@@ -112,13 +109,15 @@ public class RetryingHMSHandler implemen
// Due to reflection, the jdo exception is wrapped in
// invocationTargetException
caughtException = e.getCause();
- }
- else {
+ } else if (e.getCause() instanceof MetaException && e.getCause().getCause() != null
+ && e.getCause().getCause() instanceof javax.jdo.JDOException) {
+ // The JDOException may be wrapped further in a MetaException
+ caughtException = e.getCause().getCause();
+ } else {
LOG.error(ExceptionUtils.getStackTrace(e.getCause()));
throw e.getCause();
}
- }
- else {
+ } else {
LOG.error(ExceptionUtils.getStackTrace(e));
throw e;
}
@@ -127,8 +126,11 @@ public class RetryingHMSHandler implemen
// Due to reflection, the jdo exception is wrapped in
// invocationTargetException
caughtException = e.getCause();
- }
- else {
+ } else if (e.getCause() instanceof MetaException && e.getCause().getCause() != null
+ && e.getCause().getCause() instanceof javax.jdo.JDOException) {
+ // The JDOException may be wrapped further in a MetaException
+ caughtException = e.getCause().getCause();
+ } else {
LOG.error(ExceptionUtils.getStackTrace(e.getCause()));
throw e.getCause();
}
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/Driver.java?rev=1463091&r1=1463090&r2=1463091&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/Driver.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/Driver.java Mon Apr 1 07:00:00 2013
@@ -1148,8 +1148,8 @@ public class Driver implements CommandPr
int exitVal = tskRes.getExitVal();
if (exitVal != 0) {
if (tsk.ifRetryCmdWhenFail()) {
- if (running.size() != 0) {
- taskCleanup();
+ if (!running.isEmpty()) {
+ taskCleanup(running);
}
// in case we decided to run everything in local mode, restore the
// the jobtracker setting to its initial value
@@ -1193,8 +1193,8 @@ public class Driver implements CommandPr
}
SQLState = "08S01";
console.printError(errorMessage);
- if (running.size() != 0) {
- taskCleanup();
+ if (!running.isEmpty()) {
+ taskCleanup(running);
}
// in case we decided to run everything in local mode, restore the
// the jobtracker setting to its initial value
@@ -1359,12 +1359,18 @@ public class Driver implements CommandPr
/**
* Cleans up remaining tasks in case of failure
*/
-
- public void taskCleanup() {
- // The currently existing Shutdown hooks will be automatically called,
- // killing the map-reduce processes.
- // The non MR processes will be killed as well.
- System.exit(9);
+ public void taskCleanup(Map<TaskResult, TaskRunner> running) {
+ for (Map.Entry<TaskResult, TaskRunner> entry : running.entrySet()) {
+ if (entry.getKey().isRunning()) {
+ Task<?> task = entry.getValue().getTask();
+ try {
+ task.shutdown();
+ } catch (Exception e) {
+ console.printError("Exception on shutting down task " + task.getId() + ": " + e);
+ }
+ }
+ }
+ running.clear();
}
/**
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java?rev=1463091&r1=1463090&r2=1463091&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java Mon Apr 1 07:00:00 2013
@@ -236,8 +236,8 @@ public enum ErrorMsg {
"Fix the metadata or don't use bucketed mapjoin, by setting " +
"hive.enforce.bucketmapjoin to false."),
- JOINNODE_OUTERJOIN_MORETHAN_8(10142, "Single join node containing outer join(s) " +
- "cannot have more than 8 aliases"),
+ JOINNODE_OUTERJOIN_MORETHAN_32(10142, "Single join node containing outer join(s) " +
+ "cannot have more than 32 aliases"),
INVALID_JDO_FILTER_EXPRESSION(10043, "Invalid expression for JDO filter"),
@@ -328,6 +328,17 @@ public enum ErrorMsg {
ANALYZE_TABLE_NOSCAN_NON_NATIVE(10228, "ANALYZE TABLE NOSCAN cannot be used for "
+ "a non-native table"),
+ ANALYZE_TABLE_PARTIALSCAN_NON_NATIVE(10229, "ANALYZE TABLE PARTIALSCAN cannot be used for "
+ + "a non-native table"),
+ ANALYZE_TABLE_PARTIALSCAN_NON_RCFILE(10230, "ANALYZE TABLE PARTIALSCAN doesn't "
+ + "support non-RCfile. "),
+ ANALYZE_TABLE_PARTIALSCAN_EXTERNAL_TABLE(10231, "ANALYZE TABLE PARTIALSCAN "
+ + "doesn't support external table: "),
+ ANALYZE_TABLE_PARTIALSCAN_AGGKEY(10232, "Analyze partialscan command "
+ + "fails to construct aggregation for the partition "),
+ ANALYZE_TABLE_PARTIALSCAN_AUTOGATHER(10233, "Analyze partialscan is not allowed " +
+ "if hive.stats.autogather is set to false"),
+
SCRIPT_INIT_ERROR(20000, "Unable to initialize custom script."),
SCRIPT_IO_ERROR(20001, "An error occurred while reading or writing to your custom script. "
+ "It may have crashed with an error."),
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java?rev=1463091&r1=1463090&r2=1463091&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java Mon Apr 1 07:00:00 2013
@@ -43,6 +43,7 @@ public class QueryProperties {
boolean hasDistributeBy = false;
boolean hasClusterBy = false;
+ boolean mapJoinRemoved = false;
public boolean hasJoin() {
return hasJoin;
@@ -107,4 +108,12 @@ public class QueryProperties {
public void setHasClusterBy(boolean hasClusterBy) {
this.hasClusterBy = hasClusterBy;
}
+
+ public boolean isMapJoinRemoved() {
+ return mapJoinRemoved;
+ }
+
+ public void setMapJoinRemoved(boolean mapJoinRemoved) {
+ this.mapJoinRemoved = mapJoinRemoved;
+ }
}
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java?rev=1463091&r1=1463090&r2=1463091&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java Mon Apr 1 07:00:00 2013
@@ -21,7 +21,6 @@ package org.apache.hadoop.hive.ql.exec;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -36,7 +35,7 @@ import org.apache.hadoop.hive.ql.metadat
import org.apache.hadoop.hive.ql.plan.JoinCondDesc;
import org.apache.hadoop.hive.ql.plan.JoinDesc;
import org.apache.hadoop.hive.ql.plan.TableDesc;
-import org.apache.hadoop.hive.serde2.io.ByteWritable;
+import org.apache.hadoop.hive.serde2.io.ShortWritable;
import org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
@@ -52,40 +51,6 @@ public abstract class CommonJoinOperator
protected static final Log LOG = LogFactory.getLog(CommonJoinOperator.class
.getName());
- /**
- * IntermediateObject.
- *
- */
- public static class IntermediateObject {
- ArrayList<Object>[] objs;
- int curSize;
-
- public IntermediateObject(ArrayList<Object>[] objs, int curSize) {
- this.objs = objs;
- this.curSize = curSize;
- }
-
- public ArrayList<Object>[] getObjs() {
- return objs;
- }
-
- public int getCurSize() {
- return curSize;
- }
-
- public void pushObj(ArrayList<Object> newObj) {
- objs[curSize++] = newObj;
- }
-
- public void popObj() {
- curSize--;
- }
-
- public Object topObj() {
- return objs[curSize - 1];
- }
- }
-
protected transient int numAliases; // number of aliases
/**
* The expressions for join inputs.
@@ -97,7 +62,7 @@ public abstract class CommonJoinOperator
*/
protected transient List<ExprNodeEvaluator>[] joinFilters;
- protected transient int[][] filterMap;
+ protected transient int[][] filterMaps;
/**
* The ObjectInspectors for the join inputs.
@@ -123,14 +88,13 @@ public abstract class CommonJoinOperator
protected transient boolean[] nullsafes;
public transient boolean noOuterJoin;
- protected transient Object[] dummyObj; // for outer joins, contains the
- // potential nulls for the concerned
- // aliases
- protected transient RowContainer<ArrayList<Object>>[] dummyObjVectors; // empty
- // rows
- // for
- // each
- // table
+
+ // for outer joins, contains the potential nulls for the concerned aliases
+ protected transient ArrayList<Object>[] dummyObj;
+
+ // empty rows for each table
+ protected transient RowContainer<ArrayList<Object>>[] dummyObjVectors;
+
protected transient int totalSz; // total size of the composite object
// keys are the column names. basically this maps the position of the column
@@ -264,7 +228,7 @@ public abstract class CommonJoinOperator
joinValuesStandardObjectInspectors = JoinUtil.getStandardObjectInspectors(
joinValuesObjectInspectors,NOTSKIPBIGTABLE, tagLen);
- filterMap = conf.getFilterMap();
+ filterMaps = conf.getFilterMap();
if (noOuterJoin) {
rowContainerStandardObjectInspectors = joinValuesStandardObjectInspectors;
@@ -273,19 +237,16 @@ public abstract class CommonJoinOperator
for (Byte alias : order) {
ArrayList<ObjectInspector> rcOIs = new ArrayList<ObjectInspector>();
rcOIs.addAll(joinValuesObjectInspectors[alias]);
- // for each alias, add object inspector for boolean as the last element
+ // for each alias, add object inspector for short as the last element
rcOIs.add(
- PrimitiveObjectInspectorFactory.writableByteObjectInspector);
+ PrimitiveObjectInspectorFactory.writableShortObjectInspector);
rowContainerObjectInspectors[alias] = rcOIs;
}
rowContainerStandardObjectInspectors =
JoinUtil.getStandardObjectInspectors(rowContainerObjectInspectors,NOTSKIPBIGTABLE, tagLen);
}
-
-
-
- dummyObj = new Object[numAliases];
+ dummyObj = new ArrayList[numAliases];
dummyObjVectors = new RowContainer[numAliases];
joinEmitInterval = HiveConf.getIntVar(hconf,
@@ -309,7 +270,7 @@ public abstract class CommonJoinOperator
// add whether the row is filtered or not
// this value does not matter for the dummyObj
// because the join values are already null
- nr.add(new ByteWritable());
+ nr.add(new ShortWritable());
}
dummyObj[pos] = nr;
// there should be only 1 dummy object in the RowContainer
@@ -317,20 +278,37 @@ public abstract class CommonJoinOperator
rowContainerStandardObjectInspectors[pos],
alias, 1, spillTableDesc, conf, !hasFilter(pos), reporter);
- values.add((ArrayList<Object>) dummyObj[pos]);
+ values.add(dummyObj[pos]);
dummyObjVectors[pos] = values;
// if serde is null, the input doesn't need to be spilled out
// e.g., the output columns does not contains the input table
RowContainer rc = JoinUtil.getRowContainer(hconf,
rowContainerStandardObjectInspectors[pos],
- alias, joinCacheSize,spillTableDesc, conf, !hasFilter(pos), reporter);
+ alias, joinCacheSize, spillTableDesc, conf, !hasFilter(pos), reporter);
storage[pos] = rc;
pos++;
}
forwardCache = new Object[totalSz];
+ aliasFilterTags = new short[numAliases];
+ Arrays.fill(aliasFilterTags, (byte)0xff);
+
+ filterTags = new short[numAliases];
+ skipVectors = new boolean[numAliases][];
+ for(int i = 0; i < skipVectors.length; i++) {
+ skipVectors[i] = new boolean[i + 1];
+ }
+ intermediate = new List[numAliases];
+
+ offsets = new int[numAliases + 1];
+ int sum = 0;
+ for (int i = 0; i < numAliases; i++) {
+ offsets[i] = sum;
+ sum += joinValues[order[i]].size();
+ }
+ offsets[numAliases] = sum;
outputObjInspector = getJoinOutputObjectInspector(order,
joinValuesStandardObjectInspectors, conf);
@@ -341,16 +319,11 @@ public abstract class CommonJoinOperator
}
}
- LOG.info("JOIN "
- + outputObjInspector.getTypeName()
- + " totalsz = " + totalSz);
-
+ LOG.info("JOIN " + outputObjInspector.getTypeName() + " totalsz = " + totalSz);
}
+ transient boolean newGroupStarted = false;
-
-
-transient boolean newGroupStarted = false;
@Override
public void startGroup() throws HiveException {
LOG.trace("Join: Starting new group");
@@ -370,353 +343,267 @@ transient boolean newGroupStarted = fals
}
protected transient Byte alias;
+ protected transient Object[] forwardCache;
- transient Object[] forwardCache;
+ // pre-calculated offset values for each alias
+ protected transient int[] offsets;
- private void createForwardJoinObject(IntermediateObject intObj,
- boolean[] nullsArr) throws HiveException {
- int p = 0;
+ // a array of bitvectors where each entry denotes whether the element is to
+ // be used or not (whether it is null or not). The size of the bitvector is
+ // same as the number of inputs(aliases) under consideration currently.
+ // When all inputs are accounted for, the output is forwarded appropriately.
+ protected transient boolean[][] skipVectors;
+
+ // caches objects before constructing forward cache
+ protected transient List[] intermediate;
+
+ // filter tags for objects
+ protected transient short[] filterTags;
+
+ // ANDed value of all filter tags in current join group
+ // if any of values passes on outer join alias (which makes zero for the tag alias),
+ // it means there exists a pair for it, and no need to check outer join (just do inner join)
+ //
+ // for example, with table a, b something like,
+ // a, b = 100, 10 | 100, 20 | 100, 30
+ //
+ // the query "a FOJ b ON a.k=b.k AND a.v>0 AND b.v>20" makes values with tag
+ //
+ // a = 100, 10, 00000010 | 100, 20, 00000010 | 100, 30, 00000010 : 0/1 for 'b' (alias 1)
+ // b = 100, 10, 00000001 | 100, 20, 00000001 | 100, 30, 00000000 : 0/1 for 'a' (alias 0)
+ //
+ // which makes aliasFilterTags for a = 00000010, for b = 00000000
+ //
+ // for LO, b = 0000000(0) means there is a pair object(s) in 'b' (has no 'a'-null case)
+ // for RO, a = 000000(1)0 means there is no pair object in 'a' (has null-'b' case)
+ //
+ // result : 100, 10 + 100, 30 | 100, 20 + 100, 30 | 100, 30 + 100, 30 |
+ // N + 100, 10 | N + 100, 20
+ //
+ protected transient short[] aliasFilterTags;
+
+ // all evaluation should be processed here for valid aliasFilterTags
+ //
+ // for MapJoin, filter tag is pre-calculated in MapredLocalTask and stored with value.
+ // when reading the hashtable, MapJoinObjectValue calcuates alias filter and provide it to join
+ protected ArrayList<Object> getFilteredValue(byte alias, Object row) throws HiveException {
+ boolean hasFilter = hasFilter(alias);
+ ArrayList<Object> nr = JoinUtil.computeValues(row, joinValues[alias],
+ joinValuesObjectInspectors[alias], hasFilter);
+ if (hasFilter) {
+ short filterTag = JoinUtil.isFiltered(row, joinFilters[alias],
+ joinFilterObjectInspectors[alias], filterMaps[alias]);
+ nr.add(new ShortWritable(filterTag));
+ aliasFilterTags[alias] &= filterTag;
+ }
+ return nr;
+ }
+
+ // fill forwardCache with skipvector
+ private void createForwardJoinObject(boolean[] skip) throws HiveException {
+ Arrays.fill(forwardCache, null);
+
+ boolean forward = false;
for (int i = 0; i < numAliases; i++) {
- Byte alias = order[i];
- int sz = joinValues[alias].size();
- if (nullsArr[i]) {
- for (int j = 0; j < sz; j++) {
- forwardCache[p++] = null;
- }
- } else {
- ArrayList<Object> obj = intObj.getObjs()[i];
- for (int j = 0; j < sz; j++) {
- forwardCache[p++] = obj.get(j);
+ if (!skip[i]) {
+ for (int j = offsets[i]; j < offsets[i + 1]; j++) {
+ forwardCache[j] = intermediate[i].get(j - offsets[i]);
}
+ forward = true;
}
}
-
- forward(forwardCache, outputObjInspector);
- countAfterReport = 0;
- }
-
- private void copyOldArray(boolean[] src, boolean[] dest) {
- for (int i = 0; i < src.length; i++) {
- dest[i] = src[i];
+ if (forward) {
+ forward(forwardCache, null);
+ countAfterReport = 0;
}
}
- private ArrayList<boolean[]> joinObjectsInnerJoin(
- ArrayList<boolean[]> resNulls, ArrayList<boolean[]> inputNulls,
- ArrayList<Object> newObj, IntermediateObject intObj, int left,
- boolean newObjNull) {
- if (newObjNull) {
- return resNulls;
- }
- Iterator<boolean[]> nullsIter = inputNulls.iterator();
- while (nullsIter.hasNext()) {
- boolean[] oldNulls = nullsIter.next();
- boolean oldObjNull = oldNulls[left];
- if (!oldObjNull) {
- boolean[] newNulls = new boolean[intObj.getCurSize()];
- copyOldArray(oldNulls, newNulls);
- newNulls[oldNulls.length] = false;
- resNulls.add(newNulls);
- }
- }
- return resNulls;
- }
+ // entry point (aliasNum = 0)
+ private void genJoinObject() throws HiveException {
+ boolean rightFirst = true;
+ boolean hasFilter = hasFilter(order[0]);
+ AbstractRowContainer<ArrayList<Object>> aliasRes = storage[order[0]];
+ for (List<Object> rightObj = aliasRes.first(); rightObj != null; rightObj = aliasRes.next()) {
+ boolean rightNull = rightObj == dummyObj[0];
+ if (hasFilter) {
+ filterTags[0] = getFilterTag(rightObj);
+ }
+ skipVectors[0][0] = rightNull;
+ intermediate[0] = rightObj;
- /**
- * Implement semi join operator.
- */
- private ArrayList<boolean[]> joinObjectsLeftSemiJoin(
- ArrayList<boolean[]> resNulls, ArrayList<boolean[]> inputNulls,
- ArrayList<Object> newObj, IntermediateObject intObj, int left,
- boolean newObjNull) {
- if (newObjNull) {
- return resNulls;
- }
- Iterator<boolean[]> nullsIter = inputNulls.iterator();
- while (nullsIter.hasNext()) {
- boolean[] oldNulls = nullsIter.next();
- boolean oldObjNull = oldNulls[left];
- if (!oldObjNull) {
- boolean[] newNulls = new boolean[intObj.getCurSize()];
- copyOldArray(oldNulls, newNulls);
- newNulls[oldNulls.length] = false;
- resNulls.add(newNulls);
- }
- }
- return resNulls;
- }
-
- private ArrayList<boolean[]> joinObjectsLeftOuterJoin(
- ArrayList<boolean[]> resNulls, ArrayList<boolean[]> inputNulls,
- ArrayList<Object> newObj, IntermediateObject intObj, int left, int right,
- boolean newObjNull) {
- // newObj is null if is already null or
- // if the row corresponding to the left alias does not pass through filter
- newObjNull |= isLeftFiltered(left, right, intObj.getObjs()[left]);
-
- Iterator<boolean[]> nullsIter = inputNulls.iterator();
- while (nullsIter.hasNext()) {
- boolean[] oldNulls = nullsIter.next();
- boolean oldObjNull = oldNulls[left];
- boolean[] newNulls = new boolean[intObj.getCurSize()];
- copyOldArray(oldNulls, newNulls);
- if (oldObjNull) {
- newNulls[oldNulls.length] = true;
- } else {
- newNulls[oldNulls.length] = newObjNull;
- }
- resNulls.add(newNulls);
+ genObject(1, rightFirst, rightNull);
+ rightFirst = false;
}
- return resNulls;
}
- private ArrayList<boolean[]> joinObjectsRightOuterJoin(
- ArrayList<boolean[]> resNulls, ArrayList<boolean[]> inputNulls,
- ArrayList<Object> newObj, IntermediateObject intObj, int left, int right,
- boolean newObjNull, boolean firstRow) {
- if (newObjNull) {
- return resNulls;
- }
-
- if (inputNulls.isEmpty() && firstRow) {
- boolean[] newNulls = new boolean[intObj.getCurSize()];
- for (int i = 0; i < intObj.getCurSize() - 1; i++) {
- newNulls[i] = true;
- }
- newNulls[intObj.getCurSize() - 1] = newObjNull;
- resNulls.add(newNulls);
- return resNulls;
- }
+ // creates objects in recursive manner
+ private void genObject(int aliasNum, boolean allLeftFirst, boolean allLeftNull)
+ throws HiveException {
+ if (aliasNum < numAliases) {
- boolean allOldObjsNull = firstRow;
+ boolean[] skip = skipVectors[aliasNum];
+ boolean[] prevSkip = skipVectors[aliasNum - 1];
- Iterator<boolean[]> nullsIter = inputNulls.iterator();
- while (nullsIter.hasNext()) {
- boolean[] oldNulls = nullsIter.next();
- if (!oldNulls[left]) {
- allOldObjsNull = false;
- break;
- }
- }
+ JoinCondDesc joinCond = condn[aliasNum - 1];
+ int type = joinCond.getType();
+ int left = joinCond.getLeft();
+ int right = joinCond.getRight();
- // if the row does not pass through filter, all old Objects are null
- if (isRightFiltered(left, right, newObj)) {
- allOldObjsNull = true;
- }
- nullsIter = inputNulls.iterator();
- while (nullsIter.hasNext()) {
- boolean[] oldNulls = nullsIter.next();
- boolean oldObjNull = oldNulls[left] || allOldObjsNull;
+ // search for match in the rhs table
+ AbstractRowContainer<ArrayList<Object>> aliasRes = storage[order[aliasNum]];
- if (!oldObjNull) {
- boolean[] newNulls = new boolean[intObj.getCurSize()];
- copyOldArray(oldNulls, newNulls);
- newNulls[oldNulls.length] = newObjNull;
- resNulls.add(newNulls);
- } else if (allOldObjsNull) {
- boolean[] newNulls = new boolean[intObj.getCurSize()];
- for (int i = 0; i < intObj.getCurSize() - 1; i++) {
- newNulls[i] = true;
+ boolean done = false;
+ boolean loopAgain = false;
+ boolean tryLOForFO = type == JoinDesc.FULL_OUTER_JOIN;
+
+ boolean rightFirst = true;
+ for (List<Object> rightObj = aliasRes.first(); !done && rightObj != null;
+ rightObj = loopAgain ? rightObj : aliasRes.next(), rightFirst = loopAgain = false) {
+ System.arraycopy(prevSkip, 0, skip, 0, prevSkip.length);
+
+ boolean rightNull = rightObj == dummyObj[aliasNum];
+ if (hasFilter(order[aliasNum])) {
+ filterTags[aliasNum] = getFilterTag(rightObj);
}
- newNulls[oldNulls.length] = newObjNull;
- resNulls.add(newNulls);
- return resNulls;
- }
- }
- return resNulls;
- }
-
- private ArrayList<boolean[]> joinObjectsFullOuterJoin(
- ArrayList<boolean[]> resNulls, ArrayList<boolean[]> inputNulls,
- ArrayList<Object> newObj, IntermediateObject intObj, int left, int right,
- boolean newObjNull, boolean firstRow) {
- if (newObjNull) {
- Iterator<boolean[]> nullsIter = inputNulls.iterator();
- while (nullsIter.hasNext()) {
- boolean[] oldNulls = nullsIter.next();
- boolean[] newNulls = new boolean[intObj.getCurSize()];
- copyOldArray(oldNulls, newNulls);
- newNulls[oldNulls.length] = newObjNull;
- resNulls.add(newNulls);
- }
- return resNulls;
- }
-
- if (inputNulls.isEmpty() && firstRow) {
- boolean[] newNulls = new boolean[intObj.getCurSize()];
- for (int i = 0; i < intObj.getCurSize() - 1; i++) {
- newNulls[i] = true;
- }
- newNulls[intObj.getCurSize() - 1] = newObjNull;
- resNulls.add(newNulls);
- return resNulls;
- }
-
- boolean allOldObjsNull = firstRow;
-
- Iterator<boolean[]> nullsIter = inputNulls.iterator();
- while (nullsIter.hasNext()) {
- boolean[] oldNulls = nullsIter.next();
- if (!oldNulls[left]) {
- allOldObjsNull = false;
- break;
- }
- }
-
- // if the row does not pass through filter, all old Objects are null
- if (isRightFiltered(left, right, newObj)) {
- allOldObjsNull = true;
- }
- boolean rhsPreserved = false;
-
- nullsIter = inputNulls.iterator();
- while (nullsIter.hasNext()) {
- boolean[] oldNulls = nullsIter.next();
- // old obj is null even if the row corresponding to the left alias
- // does not pass through filter
- boolean oldObjNull = oldNulls[left] || allOldObjsNull
- || isLeftFiltered(left, right, intObj.getObjs()[left]);
- if (!oldObjNull) {
- boolean[] newNulls = new boolean[intObj.getCurSize()];
- copyOldArray(oldNulls, newNulls);
- newNulls[oldNulls.length] = newObjNull;
- resNulls.add(newNulls);
- } else if (oldObjNull) {
- boolean[] newNulls = new boolean[intObj.getCurSize()];
- copyOldArray(oldNulls, newNulls);
- newNulls[oldNulls.length] = true;
- resNulls.add(newNulls);
-
- if (allOldObjsNull && !rhsPreserved) {
- newNulls = new boolean[intObj.getCurSize()];
- for (int i = 0; i < oldNulls.length; i++) {
- newNulls[i] = true;
+ skip[right] = rightNull;
+
+ if (type == JoinDesc.INNER_JOIN) {
+ innerJoin(skip, left, right);
+ } else if (type == JoinDesc.LEFT_SEMI_JOIN) {
+ if (innerJoin(skip, left, right)) {
+ // if left-semi-join found a match, skipping the rest of the rows in the
+ // rhs table of the semijoin
+ done = true;
+ }
+ } else if (type == JoinDesc.LEFT_OUTER_JOIN ||
+ (type == JoinDesc.FULL_OUTER_JOIN && rightNull)) {
+ int result = leftOuterJoin(skip, left, right);
+ if (result < 0) {
+ continue;
+ }
+ done = result > 0;
+ } else if (type == JoinDesc.RIGHT_OUTER_JOIN ||
+ (type == JoinDesc.FULL_OUTER_JOIN && allLeftNull)) {
+ if (allLeftFirst && !rightOuterJoin(skip, left, right) ||
+ !allLeftFirst && !innerJoin(skip, left, right)) {
+ continue;
+ }
+ } else if (type == JoinDesc.FULL_OUTER_JOIN) {
+ if (tryLOForFO && leftOuterJoin(skip, left, right) > 0) {
+ loopAgain = allLeftFirst;
+ done = !loopAgain;
+ tryLOForFO = false;
+ } else if (allLeftFirst && !rightOuterJoin(skip, left, right) ||
+ !allLeftFirst && !innerJoin(skip, left, right)) {
+ continue;
}
- newNulls[oldNulls.length] = false;
- resNulls.add(newNulls);
- rhsPreserved = true;
}
+ intermediate[aliasNum] = rightObj;
+
+ // recursively call the join the other rhs tables
+ genObject(aliasNum + 1, allLeftFirst && rightFirst, allLeftNull && rightNull);
}
+ } else if (!allLeftNull) {
+ createForwardJoinObject(skipVectors[numAliases - 1]);
}
- return resNulls;
}
- /*
- * The new input is added to the list of existing inputs. Each entry in the
- * array of inputNulls denotes the entries in the intermediate object to be
- * used. The intermediate object is augmented with the new object, and list of
- * nulls is changed appropriately. The list will contain all non-nulls for a
- * inner join. The outer joins are processed appropriately.
- */
- private ArrayList<boolean[]> joinObjects(ArrayList<boolean[]> inputNulls,
- ArrayList<Object> newObj, IntermediateObject intObj, int joinPos,
- boolean firstRow) {
- ArrayList<boolean[]> resNulls = new ArrayList<boolean[]>();
- boolean newObjNull = newObj == dummyObj[joinPos] ? true : false;
- if (joinPos == 0) {
- if (newObjNull) {
- return null;
- }
- boolean[] nulls = new boolean[1];
- nulls[0] = newObjNull;
- resNulls.add(nulls);
- return resNulls;
- }
-
- int left = condn[joinPos - 1].getLeft();
- int right = condn[joinPos - 1].getRight();
- int type = condn[joinPos - 1].getType();
-
- // process all nulls for RIGHT and FULL OUTER JOINS
- if (((type == JoinDesc.RIGHT_OUTER_JOIN) || (type == JoinDesc.FULL_OUTER_JOIN))
- && !newObjNull && (inputNulls == null) && firstRow) {
- boolean[] newNulls = new boolean[intObj.getCurSize()];
- for (int i = 0; i < newNulls.length - 1; i++) {
- newNulls[i] = true;
- }
- newNulls[newNulls.length - 1] = false;
- resNulls.add(newNulls);
- return resNulls;
- }
-
- if (inputNulls == null) {
- return null;
- }
-
- if (type == JoinDesc.INNER_JOIN) {
- return joinObjectsInnerJoin(resNulls, inputNulls, newObj, intObj, left,
- newObjNull);
- } else if (type == JoinDesc.LEFT_OUTER_JOIN) {
- return joinObjectsLeftOuterJoin(resNulls, inputNulls, newObj, intObj,
- left, right, newObjNull);
- } else if (type == JoinDesc.RIGHT_OUTER_JOIN) {
- return joinObjectsRightOuterJoin(resNulls, inputNulls, newObj, intObj,
- left, right, newObjNull, firstRow);
- } else if (type == JoinDesc.LEFT_SEMI_JOIN) {
- return joinObjectsLeftSemiJoin(resNulls, inputNulls, newObj, intObj,
- left, newObjNull);
- }
-
- assert (type == JoinDesc.FULL_OUTER_JOIN);
- return joinObjectsFullOuterJoin(resNulls, inputNulls, newObj, intObj, left, right,
- newObjNull, firstRow);
- }
-
- /*
- * genObject is a recursive function. For the inputs, a array of bitvectors is
- * maintained (inputNulls) where each entry denotes whether the element is to
- * be used or not (whether it is null or not). The size of the bitvector is
- * same as the number of inputs under consideration currently. When all inputs
- * are accounted for, the output is forwarded appropriately.
- */
- private void genObject(ArrayList<boolean[]> inputNulls, int aliasNum,
- IntermediateObject intObj, boolean firstRow) throws HiveException {
- boolean childFirstRow = firstRow;
- boolean skipping = false;
+ // inner join
+ private boolean innerJoin(boolean[] skip, int left, int right) {
+ if (!isInnerJoin(skip, left, right)) {
+ Arrays.fill(skip, true);
+ return false;
+ }
+ return true;
+ }
- if (aliasNum < numAliases) {
+ // LO
+ //
+ // LEFT\RIGHT skip filtered valid
+ // skip --(1) --(1) --(1)
+ // filtered +-(1) +-(1) +-(1)
+ // valid +-(1) +-(4*) ++(2)
+ //
+ // * If right alias has any pair for left alias, continue (3)
+ // -1 for continue : has pair but not in this turn
+ // 0 for inner join (++) : join and continue LO
+ // 1 for left outer join (+-) : join and skip further LO
+ private int leftOuterJoin(boolean[] skip, int left, int right) {
+ if (skip[left] || skip[right] || !isLeftValid(left, right)) {
+ skip[right] = true;
+ return 1; // case 1
+ }
+ if (isRightValid(left, right)) {
+ return 0; // case 2
+ }
+ if (hasRightPairForLeft(left, right)) {
+ return -1; // case 3
+ }
+ skip[right] = true;
+ return 1; // case 4
+ }
- // search for match in the rhs table
- AbstractRowContainer<ArrayList<Object>> aliasRes = storage[order[aliasNum]];
+ // RO
+ //
+ // LEFT\RIGHT skip filtered valid
+ // skip --(1) -+(1) -+(1)
+ // filtered --(1) -+(1) -+(4*)
+ // valid --(1) -+(1) ++(2)
+ //
+ // * If left alias has any pair for right alias, continue (3)
+ // false for continue : has pair but not in this turn
+ private boolean rightOuterJoin(boolean[] skip, int left, int right) {
+ if (skip[left] || skip[right] || !isRightValid(left, right)) {
+ Arrays.fill(skip, 0, right, true);
+ return true; // case 1
+ }
+ if (isLeftValid(left, right)) {
+ return true; // case 2
+ }
+ if (hasLeftPairForRight(left, right)) {
+ return false; // case 3
+ }
+ Arrays.fill(skip, 0, right, true);
+ return true; // case 4
+ }
- for (ArrayList<Object> newObj = aliasRes.first(); newObj != null; newObj = aliasRes
- .next()) {
+ // If left and right aliases are all valid, two values will be inner joined,
+ private boolean isInnerJoin(boolean[] skip, int left, int right) {
+ return !skip[left] && !skip[right] &&
+ isLeftValid(left, right) && isRightValid(left, right);
+ }
- // check for skipping in case of left semi join
- if (aliasNum > 0
- && condn[aliasNum - 1].getType() == JoinDesc.LEFT_SEMI_JOIN
- && newObj != dummyObj[aliasNum]) { // successful match
- skipping = true;
- }
+ // check if left is valid
+ private boolean isLeftValid(int left, int right) {
+ return !hasFilter(left) || !JoinUtil.isFiltered(filterTags[left], right);
+ }
+
+ // check if right is valid
+ private boolean isRightValid(int left, int right) {
+ return !hasFilter(right) || !JoinUtil.isFiltered(filterTags[right], left);
+ }
- intObj.pushObj(newObj);
+ // check if any left pair exists for right objects
+ private boolean hasLeftPairForRight(int left, int right) {
+ return !JoinUtil.isFiltered(aliasFilterTags[left], right);
+ }
- // execute the actual join algorithm
- ArrayList<boolean[]> newNulls = joinObjects(inputNulls, newObj, intObj,
- aliasNum, childFirstRow);
+ // check if any right pair exists for left objects
+ private boolean hasRightPairForLeft(int left, int right) {
+ return !JoinUtil.isFiltered(aliasFilterTags[right], left);
+ }
- // recursively call the join the other rhs tables
- genObject(newNulls, aliasNum + 1, intObj, firstRow);
+ private boolean hasAnyFiltered(int alias, List<Object> row) {
+ return row == dummyObj[alias] || hasFilter(alias) && JoinUtil.hasAnyFiltered(getFilterTag(row));
+ }
- intObj.popObj();
- firstRow = false;
+ protected final boolean hasFilter(int alias) {
+ return filterMaps != null && filterMaps[alias] != null;
+ }
- // if left-semi-join found a match, skipping the rest of the rows in the
- // rhs table of the semijoin
- if (skipping) {
- break;
- }
- }
- } else {
- if (inputNulls == null) {
- return;
- }
- Iterator<boolean[]> nullsIter = inputNulls.iterator();
- while (nullsIter.hasNext()) {
- boolean[] nullsVec = nullsIter.next();
- createForwardJoinObject(intObj, nullsVec);
- }
- }
+ // get tag value from object (last of list)
+ protected final short getFilterTag(List<Object> row) {
+ return ((ShortWritable) row.get(row.size() - 1)).get();
}
/**
@@ -783,7 +670,7 @@ transient boolean newGroupStarted = fals
}
if (alw.size() == 0) {
- alw.add((ArrayList<Object>) dummyObj[i]);
+ alw.add(dummyObj[i]);
hasNulls = true;
} else if (condn[i].getPreserved()) {
preserve = true;
@@ -821,7 +708,7 @@ transient boolean newGroupStarted = fals
} else {
if (alw.size() == 0) {
hasEmpty = true;
- alw.add((ArrayList<Object>) dummyObj[i]);
+ alw.add(dummyObj[i]);
} else if (!hasEmpty && alw.size() == 1) {
if (hasAnyFiltered(alias, alw.first())) {
hasEmpty = true;
@@ -851,39 +738,11 @@ transient boolean newGroupStarted = fals
LOG.trace("called genUniqueJoinObject");
} else {
LOG.trace("calling genObject");
- genObject(null, 0, new IntermediateObject(new ArrayList[numAliases], 0),
- true);
+ genJoinObject();
LOG.trace("called genObject");
}
}
- }
-
- // returns filter result of left object by filters associated with right alias
- private boolean isLeftFiltered(int left, int right, List<Object> leftObj) {
- if (joinValues[order[left]].size() < leftObj.size()) {
- ByteWritable filter = (ByteWritable) leftObj.get(leftObj.size() - 1);
- return JoinUtil.isFiltered(filter.get(), right);
- }
- return false;
- }
-
- // returns filter result of right object by filters associated with left alias
- private boolean isRightFiltered(int left, int right, List<Object> rightObj) {
- if (joinValues[order[right]].size() < rightObj.size()) {
- ByteWritable filter = (ByteWritable) rightObj.get(rightObj.size() - 1);
- return JoinUtil.isFiltered(filter.get(), left);
- }
- return false;
- }
-
- // returns object has any filtered tag
- private boolean hasAnyFiltered(int alias, List<Object> row) {
- return row == dummyObj[alias] ||
- hasFilter(alias) && JoinUtil.hasAnyFiltered(((ByteWritable) row.get(row.size() - 1)).get());
- }
-
- protected final boolean hasFilter(int alias) {
- return filterMap != null && filterMap[alias] != null;
+ Arrays.fill(aliasFilterTags, (byte)0xff);
}
protected void reportProgress() {
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java?rev=1463091&r1=1463090&r2=1463091&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java Mon Apr 1 07:00:00 2013
@@ -45,7 +45,6 @@ import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
-import org.antlr.stringtemplate.StringTemplate;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
@@ -165,6 +164,7 @@ import org.apache.hadoop.hive.shims.Hado
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.ToolRunner;
+import org.stringtemplate.v4.ST;
/**
* DDLTask implementation.
@@ -1948,17 +1948,17 @@ public class DDLTask extends Task<DDLWor
return 0;
}
- StringTemplate createTab_stmt = new StringTemplate("CREATE $" + EXTERNAL + "$ TABLE " +
+ ST createTab_stmt = new ST("CREATE <" + EXTERNAL + "> TABLE " +
tableName + "(\n" +
- "$" + LIST_COLUMNS + "$)\n" +
- "$" + TBL_COMMENT + "$\n" +
- "$" + LIST_PARTITIONS + "$\n" +
- "$" + SORT_BUCKET + "$\n" +
- "$" + ROW_FORMAT + "$\n" +
+ "<" + LIST_COLUMNS + ">)\n" +
+ "<" + TBL_COMMENT + ">\n" +
+ "<" + LIST_PARTITIONS + ">\n" +
+ "<" + SORT_BUCKET + ">\n" +
+ "<" + ROW_FORMAT + ">\n" +
"LOCATION\n" +
- "$" + TBL_LOCATION + "$\n" +
+ "<" + TBL_LOCATION + ">\n" +
"TBLPROPERTIES (\n" +
- "$" + TBL_PROPERTIES + "$)\n");
+ "<" + TBL_PROPERTIES + ">)\n");
// For cases where the table is external
String tbl_external = "";
@@ -2115,16 +2115,16 @@ public class DDLTask extends Task<DDLWor
tbl_properties += StringUtils.join(realProps, ", \n");
}
- createTab_stmt.setAttribute(EXTERNAL, tbl_external);
- createTab_stmt.setAttribute(LIST_COLUMNS, tbl_columns);
- createTab_stmt.setAttribute(TBL_COMMENT, tbl_comment);
- createTab_stmt.setAttribute(LIST_PARTITIONS, tbl_partitions);
- createTab_stmt.setAttribute(SORT_BUCKET, tbl_sort_bucket);
- createTab_stmt.setAttribute(ROW_FORMAT, tbl_row_format);
- createTab_stmt.setAttribute(TBL_LOCATION, tbl_location);
- createTab_stmt.setAttribute(TBL_PROPERTIES, tbl_properties);
+ createTab_stmt.add(EXTERNAL, tbl_external);
+ createTab_stmt.add(LIST_COLUMNS, tbl_columns);
+ createTab_stmt.add(TBL_COMMENT, tbl_comment);
+ createTab_stmt.add(LIST_PARTITIONS, tbl_partitions);
+ createTab_stmt.add(SORT_BUCKET, tbl_sort_bucket);
+ createTab_stmt.add(ROW_FORMAT, tbl_row_format);
+ createTab_stmt.add(TBL_LOCATION, tbl_location);
+ createTab_stmt.add(TBL_PROPERTIES, tbl_properties);
- outStream.writeBytes(createTab_stmt.toString());
+ outStream.writeBytes(createTab_stmt.render());
((FSDataOutputStream) outStream).close();
outStream = null;
} catch (FileNotFoundException e) {
@@ -3955,20 +3955,15 @@ public class DDLTask extends Task<DDLWor
Table table = db.getTable(tableName, true);
- FsShell fshell = new FsShell(conf);
try {
+ // this is not transactional
for (Path location : getLocations(db, table, partSpec)) {
- fshell.run(new String[]{"-rmr", location.toString()});
- location.getFileSystem(conf).mkdirs(location);
+ FileSystem fs = location.getFileSystem(conf);
+ fs.delete(location, true);
+ fs.mkdirs(location);
}
} catch (Exception e) {
throw new HiveException(e);
- } finally {
- try {
- fshell.close();
- } catch (IOException e) {
- // ignore
- }
}
return 0;
}
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java?rev=1463091&r1=1463090&r2=1463091&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java Mon Apr 1 07:00:00 2013
@@ -25,7 +25,6 @@ import java.io.OutputStream;
import java.io.Serializable;
import java.lang.management.ManagementFactory;
import java.lang.management.MemoryMXBean;
-import java.net.URL;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -89,7 +88,6 @@ import org.apache.log4j.Appender;
import org.apache.log4j.BasicConfigurator;
import org.apache.log4j.FileAppender;
import org.apache.log4j.LogManager;
-import org.apache.log4j.PropertyConfigurator;
import org.apache.log4j.varia.NullAppender;
/**
@@ -107,6 +105,8 @@ public class ExecDriver extends Task<Map
protected static transient final Log LOG = LogFactory.getLog(ExecDriver.class);
+ private RunningJob rj;
+
/**
* Constructor when invoked from QL.
*/
@@ -358,7 +358,6 @@ public class ExecDriver extends Task<Map
initializeFiles("tmpfiles", addedFiles);
}
int returnVal = 0;
- RunningJob rj = null;
boolean noName = StringUtils.isEmpty(HiveConf.getVar(job, HiveConf.ConfVars.HADOOPJOBNAME));
if (noName) {
@@ -980,4 +979,17 @@ public class ExecDriver extends Task<Map
public void logPlanProgress(SessionState ss) throws IOException {
ss.getHiveHistory().logPlanProgress(queryPlan);
}
+
+ @Override
+ public void shutdown() {
+ super.shutdown();
+ if (rj != null) {
+ try {
+ rj.killJob();
+ } catch (Exception e) {
+ LOG.warn("failed to kill job " + rj.getID(), e);
+ }
+ rj = null;
+ }
+ }
}