You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/04/26 21:16:13 UTC
svn commit: r1476348 [2/29] - in /hive/branches/vectorization: ./ beeline/
beeline/src/java/org/apache/hive/beeline/ beeline/src/test/org/
beeline/src/test/org/apache/ beeline/src/test/org/apache/hive/
beeline/src/test/org/apache/hive/beeline/ beeline/...
Modified: hive/branches/vectorization/build.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/build.xml?rev=1476348&r1=1476347&r2=1476348&view=diff
==============================================================================
--- hive/branches/vectorization/build.xml (original)
+++ hive/branches/vectorization/build.xml Fri Apr 26 19:14:49 2013
@@ -385,7 +385,6 @@
<mkdir dir="${target.conf.dir}"/>
<mkdir dir="${target.bin.dir}"/>
<mkdir dir="${target.scripts.dir}/metastore/upgrade"/>
- <mkdir dir="${target.scripts.dir}/pdk"/>
<mkdir dir="${target.example.dir}"/>
<mkdir dir="${target.example.dir}/files"/>
<mkdir dir="${target.example.dir}/queries"/>
@@ -402,9 +401,6 @@
<copy todir="${target.scripts.dir}/metastore/upgrade">
<fileset dir="${hive.root}/metastore/scripts/upgrade" excludes="${vcs.excludes}"/>
</copy>
- <copy todir="${target.scripts.dir}/pdk">
- <fileset dir="${hive.root}/pdk/scripts" excludes="${vcs.excludes},README"/>
- </copy>
<copy file="${basedir}/conf/hive-default.xml.template" todir="${target.conf.dir}">
<filterset>
<filter token="VERSION" value="${version}"/>
@@ -492,9 +488,6 @@
<copy todir="${target.example.dir}/files" preservelastmodified="true" flatten="true">
<fileset dir="${test.data.dir}/files" includes="*.*" excludes="${vcs.excludes}"/>
</copy>
- <copy todir="${target.example.dir}/test-plugin">
- <fileset dir="${hive.root}/pdk/test-plugin"/>
- </copy>
<copy file="${basedir}/README.txt" todir="${target.dir}">
<filterset>
<filter token="VERSION" value="${version}"/>
@@ -517,14 +510,6 @@
<copy todir="${build.dir.hive}/hcatalog">
<fileset dir="${hive.root}/hcatalog/build/hcatalog-${hcatalog.version}"/>
</copy>
-
- <!-- special case because builtins compilation depends on packaging
- up everything else first -->
- <ant antfile="${hive.root}/builtins/build.xml" target="package"
- inheritAll="false" />
- <copy todir="${target.lib.dir}" preservelastmodified="true" flatten="true">
- <fileset file="${build.dir.hive}/builtins/hive-builtins-${version}.jar"/>
- </copy>
</target>
@@ -678,13 +663,12 @@
<packageset dir="hcatalog/src/java"/>
<packageset dir="cli/src/java"/>
<packageset dir="beeline/src/java"/>
+ <packageset dir="beeline/src/test"/>
<packageset dir="ql/src/java"/>
<packageset dir="ql/src/test"/>
<packageset dir="ql/src/gen/thrift/gen-javabean"/>
<packageset dir="${build.dir.hive}/ql/gen/antlr/gen-java"/>
<packageset dir="shims/src/common/java"/>
- <packageset dir="pdk/src/java"/>
- <packageset dir="builtins/src"/>
<link href="${javadoc.link.java}"/>
@@ -954,7 +938,6 @@
<sourcePath path="${hive.root}/ant" />
<sourcePath path="${hive.root}/beeline" />
- <sourcePath path="${hive.root}/builtins" />
<sourcePath path="${hive.root}/cli" />
<sourcePath path="${hive.root}/common" />
<sourcePath path="${hive.root}/contrib" />
@@ -964,14 +947,12 @@
<sourcePath path="${hive.root}/metastore" />
<sourcePath path="${hive.root}/hcatalog" />
<sourcePath path="${hive.root}/odbc" />
- <sourcePath path="${hive.root}/pdk" />
<sourcePath path="${hive.root}/ql" />
<sourcePath path="${hive.root}/serde" />
<sourcePath path="${hive.root}/service" />
<sourcePath path="${hive.root}/shims" />
<class location="${build.dir.hive}/anttasks/classes" />
<class location="${build.dir.hive}/beeline/classes" />
- <class location="${build.dir.hive}/builtins/classes" />
<class location="${build.dir.hive}/cli/classes" />
<class location="${build.dir.hive}/common/classes" />
<class location="${build.dir.hive}/contrib/classes" />
@@ -980,7 +961,6 @@
<class location="${build.dir.hive}/jdbc/classes" />
<class location="${build.dir.hive}/metastore/classes" />
<class location="${build.dir.hive}/hcatalog/classes" />
- <class location="${build.dir.hive}/pdk/classes" />
<class location="${build.dir.hive}/ql/classes" />
<class location="${build.dir.hive}/serde/classes" />
<class location="${build.dir.hive}/service/classes" />
@@ -1065,8 +1045,6 @@
todir="${mvn.jar.dir}" />
<copy file="${build.dir.hive}/beeline/hive-beeline-${version}.jar"
todir="${mvn.jar.dir}" />
- <copy file="${build.dir.hive}/builtins/hive-builtins-${version}.jar"
- todir="${mvn.jar.dir}" />
<copy file="${build.dir.hive}/cli/hive-cli-${version}.jar"
todir="${mvn.jar.dir}" />
<copy file="${build.dir.hive}/common/hive-common-${version}.jar"
@@ -1083,8 +1061,6 @@
todir="${mvn.jar.dir}" />
<copy file="${build.dir.hive}/hcatalog/hive-hcatalog-${hcatalog.version}.jar"
todir="${mvn.jar.dir}" />
- <copy file="${build.dir.hive}/pdk/hive-pdk-${version}.jar"
- todir="${mvn.jar.dir}" />
<copy file="${build.dir.hive}/ql/hive-exec-${version}.jar"
todir="${mvn.jar.dir}" />
<copy file="${build.dir.hive}/serde/hive-serde-${version}.jar"
@@ -1099,8 +1075,6 @@
tofile="${mvn.pom.dir}/hive-anttasks-${version}.pom" />
<copy file="${build.dir.hive}/beeline/pom.xml"
tofile="${mvn.pom.dir}/hive-beeline-${version}.pom" />
- <copy file="${build.dir.hive}/builtins/pom.xml"
- tofile="${mvn.pom.dir}/hive-builtins-${version}.pom" />
<copy file="${build.dir.hive}/cli/pom.xml"
tofile="${mvn.pom.dir}/hive-cli-${version}.pom" />
<copy file="${build.dir.hive}/common/pom.xml"
@@ -1117,9 +1091,7 @@
tofile="${mvn.pom.dir}/hive-metastore-${version}.pom" />
<copy file="${build.dir.hive}/hcatalog/pom.xml"
tofile="${mvn.pom.dir}/hive-hcatalog-${hcatalog.version}.pom" />
- <copy file="${build.dir.hive}/pdk/pom.xml"
- tofile="${mvn.pom.dir}/hive-pdk-${version}.pom" />
- <copy file="${build.dir.hive}/ql/pom.xml"
+ <copy file="${build.dir.hive}/ql/pom.xml"
tofile="${mvn.pom.dir}/hive-exec-${version}.pom" />
<copy file="${build.dir.hive}/serde/pom.xml"
tofile="${mvn.pom.dir}/hive-serde-${version}.pom" />
@@ -1207,9 +1179,6 @@
<param name="hive.project" value="beeline" />
</antcall>
<antcall target="maven-publish-artifact">
- <param name="hive.project" value="builtins" />
- </antcall>
- <antcall target="maven-publish-artifact">
<param name="hive.project" value="cli" />
</antcall>
<antcall target="maven-publish-artifact">
@@ -1237,9 +1206,6 @@
<param name="hive.project" value="hcatalog" />
</antcall>
<antcall target="maven-publish-artifact">
- <param name="hive.project" value="pdk" />
- </antcall>
- <antcall target="maven-publish-artifact">
<param name="hive.project" value="serde" />
</antcall>
<antcall target="maven-publish-artifact">
@@ -1383,16 +1349,6 @@
output.file="${mvn.pom.dir}/hive-hcatalog-${hcatalog.version}.pom.asc"
gpg.passphrase="${gpg.passphrase}"/>
- <!-- hive-pdk -->
- <sign-artifact
- input.file="${mvn.pom.dir}/hive-pdk-${version}.jar"
- output.file="${mvn.pom.dir}/hive-pdk-${version}.jar.asc"
- gpg.passphrase="${gpg.passphrase}"/>
- <sign-artifact
- input.file="${mvn.pom.dir}/hive-pdk-${version}.pom"
- output.file="${mvn.pom.dir}/hive-pdk-${version}.pom.asc"
- gpg.passphrase="${gpg.passphrase}"/>
-
<!-- hive-serde -->
<sign-artifact
input.file="${mvn.jar.dir}/hive-serde-${version}.jar"
Modified: hive/branches/vectorization/cli/ivy.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/cli/ivy.xml?rev=1476348&r1=1476347&r2=1476348&view=diff
==============================================================================
--- hive/branches/vectorization/cli/ivy.xml (original)
+++ hive/branches/vectorization/cli/ivy.xml Fri Apr 26 19:14:49 2013
@@ -35,7 +35,5 @@
conf="compile->default" />
<dependency org="org.apache.hive" name="hive-shims" rev="${version}"
conf="runtime" transitive="false"/>
- <dependency org="org.apache.hive" name="hive-builtins" rev="${version}"
- conf="runtime" transitive="false"/>
</dependencies>
</ivy-module>
Modified: hive/branches/vectorization/common/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/common/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java?rev=1476348&r1=1476347&r2=1476348&view=diff
==============================================================================
--- hive/branches/vectorization/common/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java (original)
+++ hive/branches/vectorization/common/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java Fri Apr 26 19:14:49 2013
@@ -31,7 +31,7 @@ public class HiveDecimal implements Comp
public static final HiveDecimal ZERO = new HiveDecimal(BigDecimal.ZERO);
- public static final int MAX_PRECISION = 36; // multiple of 9 for compact representation
+ public static final int MAX_PRECISION = 38; // fits into 128 bits
public static final HiveDecimal ONE = new HiveDecimal(BigDecimal.ONE);
Modified: hive/branches/vectorization/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1476348&r1=1476347&r2=1476348&view=diff
==============================================================================
--- hive/branches/vectorization/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/branches/vectorization/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Fri Apr 26 19:14:49 2013
@@ -131,7 +131,8 @@ public class HiveConf extends Configurat
HiveConf.ConfVars.HMSHANDLERATTEMPTS,
HiveConf.ConfVars.HMSHANDLERINTERVAL,
HiveConf.ConfVars.HMSHANDLERFORCERELOADCONF,
- HiveConf.ConfVars.METASTORE_PARTITION_NAME_WHITELIST_PATTERN
+ HiveConf.ConfVars.METASTORE_PARTITION_NAME_WHITELIST_PATTERN,
+ HiveConf.ConfVars.METASTORE_DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES
};
/**
@@ -190,7 +191,6 @@ public class HiveConf extends Configurat
PREEXECHOOKS("hive.exec.pre.hooks", ""),
POSTEXECHOOKS("hive.exec.post.hooks", ""),
ONFAILUREHOOKS("hive.exec.failure.hooks", ""),
- OPERATORHOOKS("hive.exec.operator.hooks", ""),
CLIENTSTATSPUBLISHERS("hive.client.stats.publishers", ""),
EXECPARALLEL("hive.exec.parallel", false), // parallel query launching
EXECPARALLETHREADNUMBER("hive.exec.parallel.thread.number", 8),
@@ -340,7 +340,8 @@ public class HiveConf extends Configurat
METASTORE_EXECUTE_SET_UGI("hive.metastore.execute.setugi", false),
METASTORE_PARTITION_NAME_WHITELIST_PATTERN(
"hive.metastore.partition.name.whitelist.pattern", ""),
-
+ METASTORE_DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES(
+ "hive.metastore.disallow.incompatible.col.type.changes", false),
// Default parameters for creating tables
NEWTABLEDEFAULTPARA("hive.table.parameters.default", ""),
@@ -487,11 +488,15 @@ public class HiveConf extends Configurat
HIVEUSEEXPLICITRCFILEHEADER("hive.exec.rcfile.use.explicit.header", true),
HIVEUSERCFILESYNCCACHE("hive.exec.rcfile.use.sync.cache", true),
+ // Maximum fraction of heap that can be used by ORC file writers
+ HIVE_ORC_FILE_MEMORY_POOL("hive.exec.orc.memory.pool", 0.5f), // 50%
+
HIVESKEWJOIN("hive.optimize.skewjoin", false),
HIVECONVERTJOIN("hive.auto.convert.join", true),
HIVECONVERTJOINNOCONDITIONALTASK("hive.auto.convert.join.noconditionaltask", true),
HIVECONVERTJOINNOCONDITIONALTASKTHRESHOLD("hive.auto.convert.join.noconditionaltask.size",
10000000L),
+ HIVEOPTIMIZEMAPJOINFOLLOWEDBYMR("hive.optimize.mapjoin.mapreduce", false),
HIVESKEWJOINKEY("hive.skewjoin.key", 100000),
HIVESKEWJOINMAPJOINNUMMAPTASK("hive.skewjoin.mapjoin.map.tasks", 10000),
HIVESKEWJOINMAPJOINMINSPLIT("hive.skewjoin.mapjoin.min.split", 33554432L), //32M
@@ -524,6 +529,8 @@ public class HiveConf extends Configurat
HIVE_AUTO_SORTMERGE_JOIN_BIGTABLE_SELECTOR(
"hive.auto.convert.sortmerge.join.bigtable.selection.policy",
"org.apache.hadoop.hive.ql.optimizer.AvgPartitionSizeBasedBigTableSelectorForAutoSMJ"),
+ HIVE_AUTO_SORTMERGE_JOIN_TOMAPJOIN(
+ "hive.auto.convert.sortmerge.join.to.mapjoin", false),
HIVESCRIPTOPERATORTRUST("hive.exec.script.trust", false),
HIVEROWOFFSET("hive.exec.rowoffset", false),
@@ -565,18 +572,6 @@ public class HiveConf extends Configurat
HIVE_INDEX_COMPACT_QUERY_MAX_SIZE("hive.index.compact.query.max.size", (long) 10 * 1024 * 1024 * 1024), // 10G
HIVE_INDEX_COMPACT_BINARY_SEARCH("hive.index.compact.binary.search", true),
- //Profiler
- HIVEPROFILERDBCLASS("hive.profiler.dbclass","jdbc:derby"),
- HIVEPROFILERJDBCDRIVER("hive.profiler.jdbcdriver", "org.apache.derby.jdbc.EmbeddedDriver"),
- HIVEPROFILERDBCONNECTIONSTRING("hive.profiler.dbconnectionstring",
- "jdbc:derby:;databaseName=TempProfilerStore;create=true"), // automatically create database
- // default timeout for JDBC connection
- HIVE_PROFILER_JDBC_TIMEOUT("hive.profiler.jdbc.timeout", 30),
- HIVE_PROFILER_RETRIES_MAX("hive.stats.retries.max",
- 0), // maximum # of retries to insert/select/delete the stats DB
- HIVE_PROFILER_RETRIES_WAIT("hive.stats.retries.wait",
- 3000), // # milliseconds to wait before the next retry
-
// Statistics
HIVESTATSAUTOGATHER("hive.stats.autogather", true),
HIVESTATSDBCLASS("hive.stats.dbclass",
@@ -722,8 +717,8 @@ public class HiveConf extends Configurat
HIVE_SERVER2_KERBEROS_PRINCIPAL("hive.server2.authentication.kerberos.principal", ""),
HIVE_SERVER2_PLAIN_LDAP_URL("hive.server2.authentication.ldap.url", null),
HIVE_SERVER2_PLAIN_LDAP_BASEDN("hive.server2.authentication.ldap.baseDN", null),
- HIVE_SERVER2_KERBEROS_IMPERSONATION("hive.server2.enable.impersonation", false),
HIVE_SERVER2_CUSTOM_AUTHENTICATION_CLASS("hive.server2.custom.authentication.class", null),
+ HIVE_SERVER2_ENABLE_DOAS("hive.server2.enable.doAs", true),
HIVE_CONF_RESTRICTED_LIST("hive.conf.restricted.list", null),
Modified: hive/branches/vectorization/conf/hive-default.xml.template
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/conf/hive-default.xml.template?rev=1476348&r1=1476347&r2=1476348&view=diff
==============================================================================
--- hive/branches/vectorization/conf/hive-default.xml.template (original)
+++ hive/branches/vectorization/conf/hive-default.xml.template Fri Apr 26 19:14:49 2013
@@ -290,6 +290,24 @@
</property>
<property>
+ <name>hive.metastore.disallow.incompatible.col.type.change</name>
+ <value></value>
+ <description>If true (default is false), ALTER TABLE operations which change the type of
+ a column (say STRING) to an incompatible type (say MAP<STRING, STRING>) are disallowed.
+ RCFile default serde (ColumnarSerde) serializes the values in such a way that the
+ datatypes can be converted from string to any type. The map is also serialized as
+ a string, which can be read as a string as well. However, with any binary
+ serialization, this is not true. Blocking the ALTER TABLE prevents ClassCastExceptions
+ when subsequently trying to access old partitions.
+
+ Primitive types like INT, STRING, BIGINT, etc are compatible with each other and are
+ not blocked.
+
+ See HIVE-4409 for more details.
+</description>
+</property>
+
+<property>
<name>hive.metastore.end.function.listeners</name>
<value></value>
<description>list of comma separated listeners for the end of metastore functions.</description>
@@ -842,6 +860,16 @@
</property>
<property>
+ <name>hive.optimize.mapjoin.mapreduce</name>
+ <value>false</value>
+ <description>If hive.auto.convert.join is off, this parameter does not take
+ affect. If it is on, and if there are map-join jobs followed by a map-reduce
+ job (for e.g a group by), each map-only job is merged with the following
+ map-reduce job.
+ </description>
+</property>
+
+<property>
<name>hive.script.auto.progress</name>
<value>false</value>
<description>Whether Hive Tranform/Map/Reduce Clause should automatically send progress information to TaskTracker to avoid the task getting killed because of inactivity. Hive sends progress information when the script is outputting to stderr. This option removes the need of periodically producing stderr messages, but users should be cautious because this may prevent infinite loops in the scripts to be killed by TaskTracker. </description>
@@ -983,6 +1011,20 @@
</property>
<property>
+ <name>hive.auto.convert.sortmerge.join.to.mapjoin</name>
+ <value>false</value>
+ <description>If hive.auto.convert.sortmerge.join is set to true, and a join was converted to a sort-merge join,
+ this parameter decides whether each table should be tried as a big table, and effectviely a map-join should be
+ tried. That would create a conditional task with n+1 children for a n-way join (1 child for each table as the
+ big table), and the backup task will be the sort-merge join. In some casess, a map-join would be faster than a
+ sort-merge join, if there is no advantage of having the output bucketed and sorted. For eg. if a very big sorted
+ and bucketed table with few files (say 10 files) are being joined with a very small sorter and bucketed table
+ with few files (10 files), the sort-merge join will only use 10 mappers, and a simple map-only join might be faster
+ if the complete small table can fit in memory, and a map-join can be performed.
+ </description>
+</property>
+
+<property>
<name>hive.metastore.ds.connection.url.hook</name>
<value></value>
<description>Name of the hook to use for retriving the JDO connection URL. If empty, the value in javax.jdo.option.ConnectionURL is used </description>
@@ -1123,42 +1165,6 @@
</property>
<property>
- <name>hive.profiler.dbclass</name>
- <value>jdbc:derby</value>
- <description>The default database that stores temporary hive statistics.</description>
-</property>
-
-<property>
- <name>hive.profiler.jdbcdriver</name>
- <value>org.apache.derby.jdbc.EmbeddedDriver</value>
- <description>The JDBC driver for the database that stores temporary hive statistics.</description>
-</property>
-
-<property>
- <name>hive.profiler.dbconnectionstring</name>
- <value>jdbc:derby:;databaseName=TempStatsStore;create=true</value>
- <description>The default connection string for the database that stores temporary hive statistics.</description>
-</property>
-
-<property>
- <name>hive.profiler.jdbc.timeout</name>
- <value>30</value>
- <description>Timeout value (number of seconds) used by JDBC connection and statements.</description>
-</property>
-
-<property>
- <name>hive.profiler.retries.max</name>
- <value>0</value>
- <description>Maximum number of retries when profiler publisher/aggregator got an exception updating intermediate database. Default is no tries on failures.</description>
-</property>
-
-<property>
- <name>hive.profiler.retries.wait</name>
- <value>3000</value>
- <description>The base waiting window (in milliseconds) before the next retry. The actual wait time is calculated by baseWindow * failues baseWindow * (failure 1) * (random number between [0.0,1.0]).</description>
-</property>
-
-<property>
<name>hive.stats.dbclass</name>
<value>jdbc:derby</value>
<description>The default database that stores temporary hive statistics.</description>
@@ -1877,7 +1883,6 @@
</description>
</property>
-
<property>
<name>hive.server2.authentication.ldap.baseDN</name>
<value></value>
@@ -1886,5 +1891,15 @@
</description>
</property>
+<property>
+ <name>hive.server2.enable.doAs</name>
+ <value>true</value>
+ <description>
+ Setting this property to true will have hive server2 execute
+ hive operations as the user making the calls to it.
+ </description>
+</property>
+
+
</configuration>
Modified: hive/branches/vectorization/contrib/src/test/results/clientpositive/fileformat_base64.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/contrib/src/test/results/clientpositive/fileformat_base64.q.out?rev=1476348&r1=1476347&r2=1476348&view=diff
==============================================================================
--- hive/branches/vectorization/contrib/src/test/results/clientpositive/fileformat_base64.q.out (original)
+++ hive/branches/vectorization/contrib/src/test/results/clientpositive/fileformat_base64.q.out Fri Apr 26 19:14:49 2013
@@ -44,8 +44,6 @@ PREHOOK: query: DESCRIBE EXTENDED base64
PREHOOK: type: DESCTABLE
POSTHOOK: query: DESCRIBE EXTENDED base64_test
POSTHOOK: type: DESCTABLE
-# col_name data_type comment
-
key int None
value string None
Modified: hive/branches/vectorization/contrib/src/test/results/clientpositive/serde_s3.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/contrib/src/test/results/clientpositive/serde_s3.q.out?rev=1476348&r1=1476347&r2=1476348&view=diff
==============================================================================
--- hive/branches/vectorization/contrib/src/test/results/clientpositive/serde_s3.q.out (original)
+++ hive/branches/vectorization/contrib/src/test/results/clientpositive/serde_s3.q.out Fri Apr 26 19:14:49 2013
@@ -15,8 +15,6 @@ PREHOOK: query: DESCRIBE s3log
PREHOOK: type: DESCTABLE
POSTHOOK: query: DESCRIBE s3log
POSTHOOK: type: DESCTABLE
-# col_name data_type comment
-
#### A masked pattern was here ####
bucketname string from deserializer
rdatetime string from deserializer
Modified: hive/branches/vectorization/data/files/kv8.txt
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/data/files/kv8.txt?rev=1476348&r1=1476347&r2=1476348&view=diff
==============================================================================
--- hive/branches/vectorization/data/files/kv8.txt (original)
+++ hive/branches/vectorization/data/files/kv8.txt Fri Apr 26 19:14:49 2013
@@ -1,60 +1,62 @@
-1234567890123456789012345678901234567
-1.234567890123456789012345678901234567
-12.34567890123456789012345678901234567
-123.4567890123456789012345678901234567
-1234.567890123456789012345678901234567
-123456789012345678901234567890123456
-0.12345678901234567890123456789012345
-1.23456789012345678901234567890123456
-12.3456789012345678901234567890123456
-123.456789012345678901234567890123456
-1234.56789012345678901234567890123456
-12345.6789012345678901234567890123456
-123456.789012345678901234567890123456
-1234567.89012345678901234567890123456
-12345678.9012345678901234567890123456
-123456789.012345678901234567890123456
-1234567890.12345678901234567890123456
-12345678901.2345678901234567890123456
-123456789012.345678901234567890123456
-1234567890123.45678901234567890123456
-12345678901234.5678901234567890123456
-123456789012345.678901234567890123456
-1234567890123456.78901234567890123456
-12345678901234567.8901234567890123456
-123456789012345678.901234567890123456
-1234567890123456789.01234567890123456
-12345678901234567890.1234567890123456
-123456789012345678901.234567890123456
-1234567890123456789012.34567890123456
-12345678901234567890123.4567890123456
-123456789012345678901234.567890123456
-1234567890123456789012345.67890123456
-12345678901234567890123456.7890123456
-123456789012345678901234567.890123456
-1234567890123456789012345678.90123456
-12345678901234567890123456789.0123456
-123456789012345678901234567890.123456
-1234567890123456789012345678901.23456
-12345678901234567890123456789012.3456
-123456789012345678901234567890123.456
-1234567890123456789012345678901234.56
-12345678901234567890123456789012345.6
-123456789012345678901234567890123456.0
-123456789012345678901234567890123456.00
-123456789012345678901234567890123456.000
-000123456789012345678901234567890123456.000
+123456789012345678901234567890123456789
+1.23456789012345678901234567890123456789
+12.3456789012345678901234567890123456789
+123.456789012345678901234567890123456789
+1234.56789012345678901234567890123456789
+12345678901234567890123456789012345678
+0.1234567890123456789012345678901234578
+1.2345678901234567890123456789012345678
+12.345678901234567890123456789012345678
+123.45678901234567890123456789012345678
+1234.5678901234567890123456789012345678
+12345.678901234567890123456789012345678
+123456.78901234567890123456789012345678
+1234567.8901234567890123456789012345678
+12345678.901234567890123456789012345678
+123456789.01234567890123456789012345678
+1234567890.1234567890123456789012345678
+12345678901.234567890123456789012345678
+123456789012.34567890123456789012345678
+1234567890123.4567890123456789012345678
+12345678901234.567890123456789012345678
+123456789012345.67890123456789012345678
+1234567890123456.7890123456789012345678
+12345678901234567.890123456789012345678
+123456789012345678.90123456789012345678
+1234567890123456789.0123456789012345678
+12345678901234567890.123456789012345678
+123456789012345678901.23456789012345678
+1234567890123456789012.3456789012345678
+12345678901234567890123.456789012345678
+123456789012345678901234.56789012345678
+1234567890123456789012345.6789012345678
+12345678901234567890123456.789012345678
+123456789012345678901234567.89012345678
+1234567890123456789012345678.9012345678
+12345678901234567890123456789.012345678
+123456789012345678901234567890.12345678
+1234567890123456789012345678901.2345678
+12345678901234567890123456789012.345678
+123456789012345678901234567890123.45678
+1234567890123456789012345678901234.5678
+12345678901234567890123456789012345.678
+123456789012345678901234567890123456.78
+12345678901234567890123456789012345.67.8
+12345678901234567890123456789012345678.0
+12345678901234567890123456789012345678.00
+12345678901234567890123456789012345678.000
+00012345678901234567890123456789012345678.000
+99999999999999999999999999999999999999
999999999999999999999999999999999999
-9999999999999999999999999999999999
-999999999999999999999999999999999
+99999999999999999999999999999999999
+-99999999999999999999999999999999999999
-999999999999999999999999999999999999
--9999999999999999999999999999999999
--999999999999999999999999999999999
-0000000000000000000000000000000000000000000
-0.00000000000000000000000000000000001
--0.00000000000000000000000000000000001
-0.000000000000000000000000000000000001
--0.000000000000000000000000000000000001
+-99999999999999999999999999999999999
+000000000000000000000000000000000000000000000
+0.0000000000000000000000000000000000001
+-0.0000000000000000000000000000000000001
+0.00000000000000000000000000000000000001
+-0.00000000000000000000000000000000000001
0.123456789012345
1.234567890123456
12.34567890123456
Modified: hive/branches/vectorization/eclipse-templates/.classpath
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/eclipse-templates/.classpath?rev=1476348&r1=1476347&r2=1476348&view=diff
==============================================================================
--- hive/branches/vectorization/eclipse-templates/.classpath (original)
+++ hive/branches/vectorization/eclipse-templates/.classpath Fri Apr 26 19:14:49 2013
@@ -89,7 +89,6 @@
<classpathentry kind="lib" path="build/ivy/lib/default/snappy-@snappy.version@.jar"/>
<classpathentry kind="lib" path="build/beeline/hive-beeline-@HIVE_VERSION@.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/default/tempus-fugit-@tempus-fugit.version@.jar"/>
- <classpathentry kind="lib" path="build/builtins/hive-builtins-@HIVE_VERSION@.jar"/>
<classpathentry kind="src" path="build/contrib/test/src"/>
<classpathentry kind="src" path="build/metastore/gen/antlr/gen-java"/>
<classpathentry kind="lib" path="build/testutils/hive-testutils-@HIVE_VERSION@.jar"/>
@@ -97,7 +96,6 @@
<classpathentry kind="src" path="build/ql/gen/antlr/gen-java"/>
<classpathentry kind="src" path="beeline/src/java"/>
<classpathentry kind="src" path="beeline/src/test"/>
- <classpathentry kind="src" path="builtins/src"/>
<classpathentry kind="src" path="cli/src/java"/>
<classpathentry kind="src" path="cli/src/test"/>
<classpathentry kind="src" path="common/src/java"/>
@@ -115,8 +113,6 @@
<classpathentry kind="src" path="metastore/src/java"/>
<classpathentry kind="src" path="metastore/src/model"/>
<classpathentry kind="src" path="metastore/src/test"/>
- <classpathentry kind="src" path="pdk/src/java"/>
- <classpathentry kind="src" path="pdk/test-plugin/src"/>
<classpathentry kind="src" path="ql/src/gen/thrift/gen-javabean"/>
<classpathentry kind="src" path="ql/src/gen/protobuf/gen-java"/>
<classpathentry kind="src" path="ql/src/java"/>
Modified: hive/branches/vectorization/hbase-handler/ivy.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hbase-handler/ivy.xml?rev=1476348&r1=1476347&r2=1476348&view=diff
==============================================================================
--- hive/branches/vectorization/hbase-handler/ivy.xml (original)
+++ hive/branches/vectorization/hbase-handler/ivy.xml Fri Apr 26 19:14:49 2013
@@ -36,6 +36,9 @@
</dependency>
<dependency org="com.github.stephenc.high-scale-lib" name="high-scale-lib" rev="1.1.1"
transitive="false"/>
+ <dependency org="com.yammer.metrics" name="metrics-core" rev="${metrics-core.version}">
+ <exclude org="org.slf4j" module="slf4j-api"/><!--causes a dual slf4j presence otherwise-->
+ </dependency>
<dependency org="org.codehaus.jackson" name="jackson-jaxrs" rev="${jackson.version}"/>
<dependency org="org.codehaus.jackson" name="jackson-xc" rev="${jackson.version}"/>
</dependencies>
Modified: hive/branches/vectorization/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java?rev=1476348&r1=1476347&r2=1476348&view=diff
==============================================================================
--- hive/branches/vectorization/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java (original)
+++ hive/branches/vectorization/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java Fri Apr 26 19:14:49 2013
@@ -34,6 +34,7 @@ import org.apache.hadoop.hbase.HTableDes
import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.mapred.TableMapReduceUtil;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hive.hbase.HBaseSerDe.ColumnMapping;
import org.apache.hadoop.hive.metastore.HiveMetaHook;
@@ -268,6 +269,7 @@ public class HBaseStorageHandler extends
configureTableJobProperties(tableDesc, jobProperties);
}
+ @Override
public void configureTableJobProperties(
TableDesc tableDesc,
Map<String, String> jobProperties) {
@@ -293,6 +295,17 @@ public class HBaseStorageHandler extends
}
@Override
+ public void configureJobConf(TableDesc tableDesc, JobConf jobConf) {
+ try {
+ TableMapReduceUtil.addDependencyJars(jobConf);
+ org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addDependencyJars(jobConf,
+ HBaseStorageHandler.class);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
public DecomposedPredicate decomposePredicate(
JobConf jobConf,
Deserializer deserializer,
Modified: hive/branches/vectorization/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java?rev=1476348&r1=1476347&r2=1476348&view=diff
==============================================================================
--- hive/branches/vectorization/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java (original)
+++ hive/branches/vectorization/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java Fri Apr 26 19:14:49 2013
@@ -133,7 +133,6 @@ public class LazyHBaseRow extends LazySt
boolean [] fieldsInited = getFieldInited();
if (!fieldsInited[fieldID]) {
- fieldsInited[fieldID] = true;
ByteArrayRef ref = null;
ColumnMapping colMap = columnsMapping.get(fieldID);
@@ -164,6 +163,9 @@ public class LazyHBaseRow extends LazySt
}
}
+ // Has to be set last because of HIVE-3179: NULL fields would not work otherwise
+ fieldsInited[fieldID] = true;
+
return fields[fieldID].getObject();
}
Modified: hive/branches/vectorization/hbase-handler/src/test/org/apache/hadoop/hive/hbase/HBaseTestSetup.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hbase-handler/src/test/org/apache/hadoop/hive/hbase/HBaseTestSetup.java?rev=1476348&r1=1476347&r2=1476348&view=diff
==============================================================================
--- hive/branches/vectorization/hbase-handler/src/test/org/apache/hadoop/hive/hbase/HBaseTestSetup.java (original)
+++ hive/branches/vectorization/hbase-handler/src/test/org/apache/hadoop/hive/hbase/HBaseTestSetup.java Fri Apr 26 19:14:49 2013
@@ -37,7 +37,6 @@ import org.apache.hadoop.hbase.client.HT
import org.apache.hadoop.hbase.MiniHBaseCluster;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.mapred.JobConf;
import org.apache.zookeeper.Watcher;
@@ -49,7 +48,6 @@ import org.apache.zookeeper.Watcher;
public class HBaseTestSetup extends TestSetup {
private MiniHBaseCluster hbaseCluster;
- private MiniZooKeeperCluster zooKeeperCluster;
private int zooKeeperPort;
private String hbaseRoot;
@@ -60,32 +58,36 @@ public class HBaseTestSetup extends Test
}
void preTest(HiveConf conf) throws Exception {
- if (hbaseCluster == null) {
- // We set up fixtures on demand for the first testcase, and leave
- // them allocated for reuse across all others. Then tearDown
- // will get called once at the very end after all testcases have
- // run, giving us a guaranteed opportunity to shut them down.
- setUpFixtures(conf);
- }
+
+ setUpFixtures(conf);
+
conf.set("hbase.rootdir", hbaseRoot);
conf.set("hbase.master", hbaseCluster.getMaster().getServerName().getHostAndPort());
conf.set("hbase.zookeeper.property.clientPort", Integer.toString(zooKeeperPort));
String auxJars = conf.getAuxJars();
- auxJars = ((auxJars == null) ? "" : (auxJars + ",")) + "file://"
+ auxJars = ((auxJars == null) ? "" : (auxJars + ",")) + "file:///"
+ new JobConf(conf, HBaseConfiguration.class).getJar();
- auxJars += ",file://" + new JobConf(conf, HBaseSerDe.class).getJar();
- auxJars += ",file://" + new JobConf(conf, Watcher.class).getJar();
+ auxJars += ",file:///" + new JobConf(conf, HBaseSerDe.class).getJar();
+ auxJars += ",file:///" + new JobConf(conf, Watcher.class).getJar();
conf.setAuxJars(auxJars);
}
private void setUpFixtures(HiveConf conf) throws Exception {
- conf.set("hbase.master", "local");
+ /* We are not starting zookeeper server here because
+ * QTestUtil already starts it.
+ */
+ int zkPort = conf.getInt("hive.zookeeper.client.port", -1);
+ if ((zkPort == zooKeeperPort) && (hbaseCluster != null)) {
+ return;
+ }
+ zooKeeperPort = zkPort;
String tmpdir = System.getProperty("user.dir")+"/../build/ql/tmp";
- hbaseRoot = "file://" + tmpdir + "/hbase";
+ this.tearDown();
+ conf.set("hbase.master", "local");
+
+ hbaseRoot = "file:///" + tmpdir + "/hbase";
conf.set("hbase.rootdir", hbaseRoot);
- zooKeeperCluster = new MiniZooKeeperCluster();
- zooKeeperPort = zooKeeperCluster.startup(
- new File(tmpdir, "zookeeper"));
+
conf.set("hbase.zookeeper.property.clientPort",
Integer.toString(zooKeeperPort));
Configuration hbaseConf = HBaseConfiguration.create(conf);
@@ -155,9 +157,5 @@ public class HBaseTestSetup extends Test
hbaseCluster.shutdown();
hbaseCluster = null;
}
- if (zooKeeperCluster != null) {
- zooKeeperCluster.shutdown();
- zooKeeperCluster = null;
- }
}
}
Modified: hive/branches/vectorization/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestLazyHBaseObject.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestLazyHBaseObject.java?rev=1476348&r1=1476347&r2=1476348&view=diff
==============================================================================
--- hive/branches/vectorization/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestLazyHBaseObject.java (original)
+++ hive/branches/vectorization/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestLazyHBaseObject.java Fri Apr 26 19:14:49 2013
@@ -546,6 +546,12 @@ public class TestLazyHBaseObject extends
+ "'c':null,'d':'no'}").replace("'", "\""),
SerDeUtils.getJSONString(o, oi));
+ // This is intentionally duplicated because of HIVE-3179
+ assertEquals(
+ ("{'key':'test-row','a':null,'b':['','a','',''],"
+ + "'c':null,'d':'no'}").replace("'", "\""),
+ SerDeUtils.getJSONString(o, oi));
+
kvs.clear();
kvs.add(new KeyValue(Bytes.toBytes("test-row"),
Bytes.toBytes("cfa"), Bytes.toBytes("a"), Bytes.toBytes("123")));
Modified: hive/branches/vectorization/hbase-handler/src/test/results/positive/hbase_queries.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hbase-handler/src/test/results/positive/hbase_queries.q.out?rev=1476348&r1=1476347&r2=1476348&view=diff
==============================================================================
--- hive/branches/vectorization/hbase-handler/src/test/results/positive/hbase_queries.q.out (original)
+++ hive/branches/vectorization/hbase-handler/src/test/results/positive/hbase_queries.q.out Fri Apr 26 19:14:49 2013
@@ -17,8 +17,6 @@ PREHOOK: query: DESCRIBE EXTENDED hbase_
PREHOOK: type: DESCTABLE
POSTHOOK: query: DESCRIBE EXTENDED hbase_table_1
POSTHOOK: type: DESCTABLE
-# col_name data_type comment
-
key int from deserializer
value string from deserializer
Modified: hive/branches/vectorization/hcatalog/bin/hcat
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/bin/hcat?rev=1476348&r1=1476347&r2=1476348&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/bin/hcat (original)
+++ hive/branches/vectorization/hcatalog/bin/hcat Fri Apr 26 19:14:49 2013
@@ -119,8 +119,18 @@ done
# Put external jars, hcat jar, and config file in the classpath
HADOOP_CLASSPATH=${HADOOP_CLASSPATH}:${HCAT_CLASSPATH}:${HCAT_JAR}:${HIVE_CONF_DIR}
-if [ -n "$HBASE_CONF_DIR" ] && [ -d $HBASE_CONF_DIR ]; then
- HADOOP_CLASSPATH=${HADOOP_CLASSPATH}:${HBASE_CONF_DIR}
+# Look for HBase in a BigTop-compatible way. Avoid thrift version
+# conflict with modern versions of HBase.
+HBASE_HOME=${HBASE_HOME:-"/usr/lib/hbase"}
+HBASE_CONF_DIR=${HBASE_CONF_DIR:-"${HBASE_HOME}/conf"}
+if [ -d ${HBASE_HOME} ] ; then
+ for jar in $(find $HBASE_HOME -name *.jar -not -name thrift\*.jar); do
+ HBASE_CLASSPATH=$HBASE_CLASSPATH:${jar}
+ done
+ export HADOOP_CLASSPATH="${HADOOP_CLASSPATH}:${HBASE_CLASSPATH}"
+fi
+if [ -d $HBASE_CONF_DIR ] ; then
+ HADOOP_CLASSPATH="${HADOOP_CLASSPATH}:${HBASE_CONF_DIR}"
fi
export HADOOP_CLASSPATH=$HADOOP_CLASSPATH
Modified: hive/branches/vectorization/hcatalog/build-support/ant/deploy.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/build-support/ant/deploy.xml?rev=1476348&r1=1476347&r2=1476348&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/build-support/ant/deploy.xml (original)
+++ hive/branches/vectorization/hcatalog/build-support/ant/deploy.xml Fri Apr 26 19:14:49 2013
@@ -21,6 +21,8 @@
<project xmlns:artifact="artifact:org.apache.maven.artifact.ant">
+ <loadproperties srcfile="${path.to.basedir}/build.properties"/>
+
<macrodef name="_sign">
<attribute name="inputFile"/>
<sequential>
@@ -38,6 +40,35 @@
</sequential>
</macrodef>
+ <macrodef name="_mvnpublish">
+ <attribute name="module"/>
+ <attribute name="jarname" default="@{module}"/>
+ <sequential>
+ <echo message="Installing local artifact for maven : @{module}" />
+ <artifact:install file="${path.to.basedir}/../build/@{module}/hive-@{jarname}-${hive.version}.jar">
+ <artifact:pom file="${path.to.basedir}/../build/@{module}/pom.xml"/>
+ <artifact:localRepository path="${mvn.local.repo}"/>
+ </artifact:install>
+ </sequential>
+ </macrodef>
+
+ <target name="hive-mvn-publish">
+ <!-- Pick built jars for all the hive artifacts and install them to local maven cache -->
+ <_mvnpublish module="shims" />
+ <_mvnpublish module="common" />
+ <_mvnpublish module="serde" />
+ <_mvnpublish module="metastore" />
+ <_mvnpublish module="ql" jarname="exec"/>
+ <_mvnpublish module="contrib" />
+ <_mvnpublish module="service" />
+ <_mvnpublish module="cli" />
+ <_mvnpublish module="jdbc" />
+ <_mvnpublish module="beeline" />
+ <_mvnpublish module="hwi" />
+ <_mvnpublish module="hbase-handler" />
+ <_mvnpublish module="testutils" />
+ </target>
+
<target name="mvn-init" unless="mvn-init.complete">
<echo message="${ant.project.name}"/>
<get src="${mvnrepo}/org/apache/maven/maven-ant-tasks/${maven-ant-tasks.version}/maven-ant-tasks-${maven-ant-tasks.version}.jar"
@@ -47,6 +78,7 @@
<taskdef resource="org/apache/maven/artifact/ant/antlib.xml"
uri="artifact:org.apache.maven.artifact.ant"
classpath="${path.to.basedir}/build/maven-ant-tasks-${maven-ant-tasks.version}.jar"/>
+ <antcall target="hive-mvn-publish" />
<artifact:dependencies>
<dependency groupId="org.apache.hcatalog" artifactId="hcatalog" version="${hcatalog.version}" scope="system" systemPath="${path.to.basedir}/pom.xml"/>
</artifact:dependencies>
Modified: hive/branches/vectorization/hcatalog/build.properties
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/build.properties?rev=1476348&r1=1476347&r2=1476348&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/build.properties (original)
+++ hive/branches/vectorization/hcatalog/build.properties Fri Apr 26 19:14:49 2013
@@ -15,7 +15,8 @@
# specific language governing permissions and limitations
# under the License.
-hcatalog.version=0.11.0-SNAPSHOT
+hive.version=0.12.0-SNAPSHOT
+hcatalog.version=${hive.version}
jar.name=${ant.project.name}-${hcatalog.version}.jar
hcatalog.jar=${ant.project.name}-${hcatalog.version}.jar
hcatalog.core.jar=${ant.project.name}-core-${hcatalog.version}.jar
Modified: hive/branches/vectorization/hcatalog/build.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/build.xml?rev=1476348&r1=1476347&r2=1476348&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/build.xml (original)
+++ hive/branches/vectorization/hcatalog/build.xml Fri Apr 26 19:14:49 2013
@@ -210,7 +210,8 @@
noqualifier="all"
windowtitle="HCatalog ${hcatalog.version} API"
doctitle="HCatalog ${hcatalog.version} API"
- failonerror="true">
+ failonerror="true"
+ useexternalfile="yes">
<packageset dir="core/src/main/java"/>
<packageset dir="hcatalog-pig-adapter/src/main/java"/>
<packageset dir="server-extensions/src/main/java"/>
Modified: hive/branches/vectorization/hcatalog/core/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/core/pom.xml?rev=1476348&r1=1476347&r2=1476348&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/core/pom.xml (original)
+++ hive/branches/vectorization/hcatalog/core/pom.xml Fri Apr 26 19:14:49 2013
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.hcatalog</groupId>
<artifactId>hcatalog</artifactId>
- <version>0.11.0-SNAPSHOT</version>
+ <version>0.12.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
@@ -32,7 +32,6 @@
<groupId>org.apache.hcatalog</groupId>
<artifactId>hcatalog-core</artifactId>
<packaging>jar</packaging>
- <version>0.11.0-SNAPSHOT</version>
<name>hcatalog-core</name>
<url>http://maven.apache.org</url>
@@ -61,12 +60,6 @@
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
- <artifactId>hive-builtins</artifactId>
- <version>${hive.version}</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hive</groupId>
<artifactId>hive-cli</artifactId>
<version>${hive.version}</version>
<scope>compile</scope>
Modified: hive/branches/vectorization/hcatalog/core/src/main/java/org/apache/hcatalog/mapreduce/HCatStorageHandler.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/core/src/main/java/org/apache/hcatalog/mapreduce/HCatStorageHandler.java?rev=1476348&r1=1476347&r2=1476348&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/core/src/main/java/org/apache/hcatalog/mapreduce/HCatStorageHandler.java (original)
+++ hive/branches/vectorization/hcatalog/core/src/main/java/org/apache/hcatalog/mapreduce/HCatStorageHandler.java Fri Apr 26 19:14:49 2013
@@ -22,8 +22,8 @@ package org.apache.hcatalog.mapreduce;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.ql.metadata.DefaultStorageHandler;
import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler;
import org.apache.hadoop.hive.ql.plan.TableDesc;
import org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider;
import org.apache.hadoop.mapred.OutputFormat;
@@ -32,7 +32,7 @@ import org.apache.hadoop.mapred.OutputFo
* The abstract Class HCatStorageHandler would server as the base class for all
* the storage handlers required for non-native tables in HCatalog.
*/
-public abstract class HCatStorageHandler implements HiveStorageHandler {
+public abstract class HCatStorageHandler extends DefaultStorageHandler {
//TODO move this to HiveStorageHandler
Modified: hive/branches/vectorization/hcatalog/hcatalog-pig-adapter/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/hcatalog-pig-adapter/pom.xml?rev=1476348&r1=1476347&r2=1476348&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/hcatalog-pig-adapter/pom.xml (original)
+++ hive/branches/vectorization/hcatalog/hcatalog-pig-adapter/pom.xml Fri Apr 26 19:14:49 2013
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.hcatalog</groupId>
<artifactId>hcatalog</artifactId>
- <version>0.11.0-SNAPSHOT</version>
+ <version>0.12.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
@@ -32,7 +32,6 @@
<groupId>org.apache.hcatalog</groupId>
<artifactId>hcatalog-pig-adapter</artifactId>
<packaging>jar</packaging>
- <version>0.11.0-SNAPSHOT</version>
<name>hcatalog-pig-adapter</name>
<url>http://maven.apache.org</url>
Modified: hive/branches/vectorization/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorer.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorer.java?rev=1476348&r1=1476347&r2=1476348&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorer.java (original)
+++ hive/branches/vectorization/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorer.java Fri Apr 26 19:14:49 2013
@@ -429,9 +429,9 @@ public class TestHCatStorer extends HCat
driver.getResults(res);
Iterator<String> itr = res.iterator();
- Assert.assertEquals("0\tNULL\tNULL\tNULL\tNULL\tnull\tnull", itr.next());
- Assert.assertEquals("NULL\t4.2\t2.2\t4\tlets hcat\tbinary-data\tnull", itr.next());
- Assert.assertEquals("3\t6.2999997\t3.3000000000000003\t6\tlets hcat\tbinary-data\tnull", itr.next());
+ Assert.assertEquals("0\tNULL\tNULL\tNULL\tNULL\tNULL\tNULL", itr.next());
+ Assert.assertEquals("NULL\t4.2\t2.2\t4\tlets hcat\tbinary-data\tNULL", itr.next());
+ Assert.assertEquals("3\t6.2999997\t3.3000000000000003\t6\tlets hcat\tbinary-data\tNULL", itr.next());
Assert.assertFalse(itr.hasNext());
server.registerQuery("B = load 'junit_unparted' using " + HCatLoader.class.getName() + ";");
Modified: hive/branches/vectorization/hcatalog/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/pom.xml?rev=1476348&r1=1476347&r2=1476348&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/pom.xml (original)
+++ hive/branches/vectorization/hcatalog/pom.xml Fri Apr 26 19:14:49 2013
@@ -30,7 +30,7 @@
<hadoop23.version>2.0.3-alpha</hadoop23.version>
<hbase.version>0.94.5</hbase.version>
<hcatalog.version>${project.version}</hcatalog.version>
- <hive.version>0.11.0-SNAPSHOT</hive.version>
+ <hive.version>${project.version}</hive.version>
<jackson.version>1.9.2</jackson.version>
<jersey.version>1.14</jersey.version>
<jetty.webhcat.version>7.6.0.v20120127</jetty.webhcat.version>
@@ -42,16 +42,10 @@
<metrics-core.version>2.1.2</metrics-core.version>
</properties>
- <parent>
- <groupId>org.apache</groupId>
- <artifactId>apache</artifactId>
- <version>11</version>
- </parent>
-
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hcatalog</groupId>
<artifactId>hcatalog</artifactId>
- <version>0.11.0-SNAPSHOT</version>
+ <version>0.12.0-SNAPSHOT</version>
<packaging>pom</packaging>
<build>
Modified: hive/branches/vectorization/hcatalog/server-extensions/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/server-extensions/pom.xml?rev=1476348&r1=1476347&r2=1476348&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/server-extensions/pom.xml (original)
+++ hive/branches/vectorization/hcatalog/server-extensions/pom.xml Fri Apr 26 19:14:49 2013
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.hcatalog</groupId>
<artifactId>hcatalog</artifactId>
- <version>0.11.0-SNAPSHOT</version>
+ <version>0.12.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
@@ -32,7 +32,6 @@
<groupId>org.apache.hcatalog</groupId>
<artifactId>hcatalog-server-extensions</artifactId>
<packaging>jar</packaging>
- <version>0.11.0-SNAPSHOT</version>
<name>server-extensions</name>
<url>http://maven.apache.org</url>
Modified: hive/branches/vectorization/hcatalog/storage-handlers/hbase/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/storage-handlers/hbase/pom.xml?rev=1476348&r1=1476347&r2=1476348&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/storage-handlers/hbase/pom.xml (original)
+++ hive/branches/vectorization/hcatalog/storage-handlers/hbase/pom.xml Fri Apr 26 19:14:49 2013
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.hcatalog</groupId>
<artifactId>hcatalog</artifactId>
- <version>0.11.0-SNAPSHOT</version>
+ <version>0.12.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
@@ -32,7 +32,6 @@
<groupId>org.apache.hcatalog</groupId>
<artifactId>hbase-storage-handler</artifactId>
<packaging>jar</packaging>
- <version>0.11.0-SNAPSHOT</version>
<name>hbase-storage-handler</name>
<url>http://maven.apache.org</url>
Modified: hive/branches/vectorization/hcatalog/webhcat/java-client/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/webhcat/java-client/pom.xml?rev=1476348&r1=1476347&r2=1476348&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/webhcat/java-client/pom.xml (original)
+++ hive/branches/vectorization/hcatalog/webhcat/java-client/pom.xml Fri Apr 26 19:14:49 2013
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.hcatalog</groupId>
<artifactId>hcatalog</artifactId>
- <version>0.11.0-SNAPSHOT</version>
+ <version>0.12.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
@@ -32,7 +32,6 @@
<groupId>org.apache.hcatalog</groupId>
<artifactId>webhcat-java-client</artifactId>
<packaging>jar</packaging>
- <version>0.11.0-SNAPSHOT</version>
<name>webhcat-java-client</name>
<url>http://maven.apache.org</url>
Modified: hive/branches/vectorization/hcatalog/webhcat/svr/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/webhcat/svr/pom.xml?rev=1476348&r1=1476347&r2=1476348&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/webhcat/svr/pom.xml (original)
+++ hive/branches/vectorization/hcatalog/webhcat/svr/pom.xml Fri Apr 26 19:14:49 2013
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.hcatalog</groupId>
<artifactId>hcatalog</artifactId>
- <version>0.11.0-SNAPSHOT</version>
+ <version>0.12.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
@@ -32,7 +32,6 @@
<groupId>org.apache.hcatalog</groupId>
<artifactId>webhcat</artifactId>
<packaging>jar</packaging>
- <version>0.11.0-SNAPSHOT</version>
<name>webhcat</name>
<url>http://maven.apache.org</url>
Modified: hive/branches/vectorization/hcatalog/webhcat/svr/src/main/bin/webhcat_config.sh
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/webhcat/svr/src/main/bin/webhcat_config.sh?rev=1476348&r1=1476347&r2=1476348&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/webhcat/svr/src/main/bin/webhcat_config.sh (original)
+++ hive/branches/vectorization/hcatalog/webhcat/svr/src/main/bin/webhcat_config.sh Fri Apr 26 19:14:49 2013
@@ -64,6 +64,8 @@ fi
# Allow alternate conf dir location.
if [ -e "${WEBHCAT_PREFIX}/etc/webhcat/webhcat-env.sh" ]; then
DEFAULT_CONF_DIR=${WEBHCAT_PREFIX}/"etc/webhcat"
+elif [ -e "${WEBHCAT_PREFIX}/conf/webhcat-env.sh" ]; then
+ DEFAULT_CONF_DIR=${WEBHCAT_PREFIX}/"conf"
else
DEFAULT_CONF_DIR="/etc/webhcat"
fi
Modified: hive/branches/vectorization/hwi/ivy.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hwi/ivy.xml?rev=1476348&r1=1476347&r2=1476348&view=diff
==============================================================================
--- hive/branches/vectorization/hwi/ivy.xml (original)
+++ hive/branches/vectorization/hwi/ivy.xml Fri Apr 26 19:14:49 2013
@@ -31,8 +31,6 @@
<dependency org="org.mortbay.jetty" name="jetty" rev="${jetty.version}" />
<!-- Test Dependencies -->
- <dependency org="org.apache.hive" name="hive-builtins" rev="${version}"
- conf="test->default" transitive="false"/>
<dependency org="commons-httpclient" name="commons-httpclient" rev="${commons-httpclient.version}"
conf="test->default"
transitive="false"/>
Modified: hive/branches/vectorization/hwi/src/test/org/apache/hadoop/hive/hwi/TestHWISessionManager.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hwi/src/test/org/apache/hadoop/hive/hwi/TestHWISessionManager.java?rev=1476348&r1=1476347&r2=1476348&view=diff
==============================================================================
--- hive/branches/vectorization/hwi/src/test/org/apache/hadoop/hive/hwi/TestHWISessionManager.java (original)
+++ hive/branches/vectorization/hwi/src/test/org/apache/hadoop/hive/hwi/TestHWISessionManager.java Fri Apr 26 19:14:49 2013
@@ -121,15 +121,10 @@ public class TestHWISessionManager exten
ArrayList<ArrayList<String>> searchBlockRes = searchItem.getResultBucket();
- // "describe [table_name]" result format
- // first line should be format name:
- // "# col_name data_type comment"
- // second line is empty
- // the following lines contain the values
- String resLine = searchBlockRes.get(0).get(2);
+ String resLine = searchBlockRes.get(0).get(0);
assertEquals(true, resLine.contains("key"));
assertEquals(true, resLine.contains("int"));
- String resLine2 = searchBlockRes.get(0).get(3);
+ String resLine2 = searchBlockRes.get(0).get(1);
assertEquals(true, resLine2.contains("value"));
assertEquals(true, resLine2.contains("string"));
Modified: hive/branches/vectorization/ivy/libraries.properties
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ivy/libraries.properties?rev=1476348&r1=1476347&r2=1476348&view=diff
==============================================================================
--- hive/branches/vectorization/ivy/libraries.properties (original)
+++ hive/branches/vectorization/ivy/libraries.properties Fri Apr 26 19:14:49 2013
@@ -43,9 +43,8 @@ commons-logging.version=1.0.4
commons-logging-api.version=1.0.4
commons-pool.version=1.5.4
derby.version=10.4.2.0
-guava-hadoop20.version=r09
-guava-hadoop23.version=11.0.2
-hbase.version=0.92.0
+guava.version=11.0.2
+hbase.version=0.94.6.1
jackson.version=1.8.8
javaewah.version=0.3.2
jdo-api.version=2.3-ec
@@ -67,5 +66,6 @@ ST4.version=4.0.4
tempus-fugit.version=1.1
snappy.version=0.2
velocity.version=1.5
+metrics-core.version=2.1.2
zookeeper.version=3.4.3
javolution.version=5.5.1
Modified: hive/branches/vectorization/jdbc/src/test/org/apache/hadoop/hive/jdbc/TestJdbcDriver.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/jdbc/src/test/org/apache/hadoop/hive/jdbc/TestJdbcDriver.java?rev=1476348&r1=1476347&r2=1476348&view=diff
==============================================================================
--- hive/branches/vectorization/jdbc/src/test/org/apache/hadoop/hive/jdbc/TestJdbcDriver.java (original)
+++ hive/branches/vectorization/jdbc/src/test/org/apache/hadoop/hive/jdbc/TestJdbcDriver.java Fri Apr 26 19:14:49 2013
@@ -766,17 +766,6 @@ public class TestJdbcDriver extends Test
assertNotNull("Statement is null", stmt);
ResultSet res = stmt.executeQuery("describe " + tableName);
-
- // "describe [table_name]" result format
- // first line should be format name:
- // "# col_name data_type comment"
- // second line is empty
- // the following lines contain the values
- res.next();
- assertEquals(true, res.getString(1).contains("col_name"));
- assertEquals(true, res.getString(2).contains("data_type"));
- assertEquals(true, res.getString(3).contains("comment"));
- res.next();
res.next();
assertEquals(true, res.getString(1).contains("under_col"));
assertEquals(true, res.getString(2).contains("int"));
Modified: hive/branches/vectorization/jdbc/src/test/org/apache/hive/jdbc/TestJdbcDriver2.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/jdbc/src/test/org/apache/hive/jdbc/TestJdbcDriver2.java?rev=1476348&r1=1476347&r2=1476348&view=diff
==============================================================================
--- hive/branches/vectorization/jdbc/src/test/org/apache/hive/jdbc/TestJdbcDriver2.java (original)
+++ hive/branches/vectorization/jdbc/src/test/org/apache/hive/jdbc/TestJdbcDriver2.java Fri Apr 26 19:14:49 2013
@@ -817,9 +817,6 @@ public class TestJdbcDriver2 extends Tes
ResultSet res = stmt.executeQuery("describe " + tableName);
- res.next(); // skip header 1
- res.next(); // skip header 2
-
res.next();
assertEquals("Column name 'under_col' not found", "under_col", res.getString(1).trim());
assertEquals("Column type 'under_col' for column under_col not found", "int", res
Modified: hive/branches/vectorization/metastore/if/hive_metastore.thrift
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/metastore/if/hive_metastore.thrift?rev=1476348&r1=1476347&r2=1476348&view=diff
==============================================================================
--- hive/branches/vectorization/metastore/if/hive_metastore.thrift (original)
+++ hive/branches/vectorization/metastore/if/hive_metastore.thrift Fri Apr 26 19:14:49 2013
@@ -131,11 +131,16 @@ struct Order {
2: i32 order // asc(1) or desc(0)
}
+// Workaround for HIVE-4322
+struct SkewedValueList {
+ 1: list<string> skewedValueList
+}
+
// this object holds all the information about skewed table
struct SkewedInfo {
1: list<string> skewedColNames, // skewed column names
2: list<list<string>> skewedColValues, //skewed values
- 3: map<list<string>, string> skewedColValueLocationMaps, //skewed value to location mappings
+ 3: map<SkewedValueList, string> skewedColValueLocationMaps, //skewed value to location mappings
}
// this object holds all the information about physical storage of the data belonging to a table
@@ -450,6 +455,10 @@ service ThriftHiveMetastore extends fb30
throws(1:NoSuchObjectException o1, 2:MetaException o2)
Partition get_partition(1:string db_name, 2:string tbl_name, 3:list<string> part_vals)
throws(1:MetaException o1, 2:NoSuchObjectException o2)
+ Partition exchange_partition(1:map<string, string> partitionSpecs, 2:string source_db,
+ 3:string source_table_name, 4:string dest_db, 5:string dest_table_name)
+ throws(1:MetaException o1, 2:NoSuchObjectException o2, 3:InvalidObjectException o3,
+ 4:InvalidInputException o4)
Partition get_partition_with_auth(1:string db_name, 2:string tbl_name, 3:list<string> part_vals,
4: string user_name, 5: list<string> group_names) throws(1:MetaException o1, 2:NoSuchObjectException o2)