You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by na...@apache.org on 2010/03/23 23:29:44 UTC
svn commit: r926818 - in /hadoop/hive/trunk: ./
contrib/src/test/results/clientnegative/ hbase-handler/src/test/queries/
hbase-handler/src/test/results/ ql/src/java/org/apache/hadoop/hive/ql/io/
Author: namit
Date: Tue Mar 23 22:29:44 2010
New Revision: 926818
URL: http://svn.apache.org/viewvc?rev=926818&view=rev
Log:
HIVE-1257. join between hbase and other tables does not work
(John Sichi via namit)
Added:
hadoop/hive/trunk/hbase-handler/src/test/queries/hbase_joins.q
hadoop/hive/trunk/hbase-handler/src/test/results/hbase_joins.q.out
Modified:
hadoop/hive/trunk/CHANGES.txt
hadoop/hive/trunk/contrib/src/test/results/clientnegative/serde_regex.q.out
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=926818&r1=926817&r2=926818&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Tue Mar 23 22:29:44 2010
@@ -291,6 +291,9 @@ Trunk - Unreleased
HIVE-1268. Cannot start metastore thrift server on a specific port
(bc Wong via namit)
+ HIVE-1257. join between hbase and other tables does not work
+ (John Sichi via namit)
+
Release 0.5.0 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/hive/trunk/contrib/src/test/results/clientnegative/serde_regex.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/contrib/src/test/results/clientnegative/serde_regex.q.out?rev=926818&r1=926817&r2=926818&view=diff
==============================================================================
--- hadoop/hive/trunk/contrib/src/test/results/clientnegative/serde_regex.q.out (original)
+++ hadoop/hive/trunk/contrib/src/test/results/clientnegative/serde_regex.q.out Tue Mar 23 22:29:44 2010
@@ -51,12 +51,12 @@ STAGE PLANS:
columns: host string, identity string, user string, time string, request string, status int, size int, referer string, agent string
if not exists: false
input format: org.apache.hadoop.mapred.TextInputFormat
- serde properties:
- input.regex ([^ ]*) ([^ ]*) ([^ ]*) (-|\[[^\]]*\]) ([^ "]*|"[^"]*") (-|[0-9]*) (-|[0-9]*)(?: ([^ "]*|"[^"]*") ([^ "]*|"[^"]*"))?
- output.format.string %1$s %2$s %3$s %4$s %5$s %6$s %7$s %8$s %9$s
# buckets: -1
output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat
serde name: org.apache.hadoop.hive.contrib.serde2.RegexSerDe
+ serde properties:
+ input.regex ([^ ]*) ([^ ]*) ([^ ]*) (-|\[[^\]]*\]) ([^ "]*|"[^"]*") (-|[0-9]*) (-|[0-9]*)(?: ([^ "]*|"[^"]*") ([^ "]*|"[^"]*"))?
+ output.format.string %1$s %2$s %3$s %4$s %5$s %6$s %7$s %8$s %9$s
name: serde_regex
isExternal: false
Added: hadoop/hive/trunk/hbase-handler/src/test/queries/hbase_joins.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/hbase-handler/src/test/queries/hbase_joins.q?rev=926818&view=auto
==============================================================================
--- hadoop/hive/trunk/hbase-handler/src/test/queries/hbase_joins.q (added)
+++ hadoop/hive/trunk/hbase-handler/src/test/queries/hbase_joins.q Tue Mar 23 22:29:44 2010
@@ -0,0 +1,64 @@
+DROP TABLE users;
+DROP TABLE states;
+DROP TABLE countries;
+
+-- From HIVE-1257
+
+CREATE TABLE users(key string, state string, country string, country_id int)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "info:state,info:country,info:country_id"
+);
+
+CREATE TABLE states(key string, name string)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "state:name"
+);
+
+CREATE TABLE countries(key string, name string, country string, country_id int)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "info:name,info:country,info:country_id"
+);
+
+INSERT OVERWRITE TABLE users SELECT 'user1', 'IA', 'USA', 0
+FROM src WHERE key=100;
+
+INSERT OVERWRITE TABLE states SELECT 'IA', 'Iowa'
+FROM src WHERE key=100;
+
+INSERT OVERWRITE TABLE countries SELECT 'USA', 'United States', 'USA', 1
+FROM src WHERE key=100;
+
+set hive.input.format = org.apache.hadoop.hive.ql.io.HiveInputFormat;
+
+SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c
+ON (u.country = c.key);
+
+SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c
+ON (u.country = c.country);
+
+SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c
+ON (u.country_id = c.country_id);
+
+SELECT u.key, u.state, s.name FROM users u JOIN states s
+ON (u.state = s.key);
+
+set hive.input.format = org.apache.hadoop.hive.ql.io.CombineHiveInputFormat;
+
+SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c
+ON (u.country = c.key);
+
+SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c
+ON (u.country = c.country);
+
+SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c
+ON (u.country_id = c.country_id);
+
+SELECT u.key, u.state, s.name FROM users u JOIN states s
+ON (u.state = s.key);
+
+DROP TABLE users;
+DROP TABLE states;
+DROP TABLE countries;
Added: hadoop/hive/trunk/hbase-handler/src/test/results/hbase_joins.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/hbase-handler/src/test/results/hbase_joins.q.out?rev=926818&view=auto
==============================================================================
--- hadoop/hive/trunk/hbase-handler/src/test/results/hbase_joins.q.out (added)
+++ hadoop/hive/trunk/hbase-handler/src/test/results/hbase_joins.q.out Tue Mar 23 22:29:44 2010
@@ -0,0 +1,202 @@
+PREHOOK: query: DROP TABLE users
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE users
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE states
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE states
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE countries
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE countries
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: -- From HIVE-1257
+
+CREATE TABLE users(key string, state string, country string, country_id int)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "info:state,info:country,info:country_id"
+)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: -- From HIVE-1257
+
+CREATE TABLE users(key string, state string, country string, country_id int)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "info:state,info:country,info:country_id"
+)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@users
+PREHOOK: query: CREATE TABLE states(key string, name string)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "state:name"
+)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE states(key string, name string)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "state:name"
+)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@states
+PREHOOK: query: CREATE TABLE countries(key string, name string, country string, country_id int)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "info:name,info:country,info:country_id"
+)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE countries(key string, name string, country string, country_id int)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "info:name,info:country,info:country_id"
+)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@countries
+PREHOOK: query: INSERT OVERWRITE TABLE users SELECT 'user1', 'IA', 'USA', 0
+FROM src WHERE key=100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@users
+POSTHOOK: query: INSERT OVERWRITE TABLE users SELECT 'user1', 'IA', 'USA', 0
+FROM src WHERE key=100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@users
+PREHOOK: query: INSERT OVERWRITE TABLE states SELECT 'IA', 'Iowa'
+FROM src WHERE key=100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@states
+POSTHOOK: query: INSERT OVERWRITE TABLE states SELECT 'IA', 'Iowa'
+FROM src WHERE key=100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@states
+PREHOOK: query: INSERT OVERWRITE TABLE countries SELECT 'USA', 'United States', 'USA', 1
+FROM src WHERE key=100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@countries
+POSTHOOK: query: INSERT OVERWRITE TABLE countries SELECT 'USA', 'United States', 'USA', 1
+FROM src WHERE key=100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@countries
+PREHOOK: query: SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c
+ON (u.country = c.key)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@users
+PREHOOK: Input: default@countries
+PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-23_12-31-19_117_4712777556867957470/10000
+POSTHOOK: query: SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c
+ON (u.country = c.key)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@users
+POSTHOOK: Input: default@countries
+POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-23_12-31-19_117_4712777556867957470/10000
+user1 USA United States USA
+PREHOOK: query: SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c
+ON (u.country = c.country)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@users
+PREHOOK: Input: default@countries
+PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-23_12-31-25_489_4149172260159935049/10000
+POSTHOOK: query: SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c
+ON (u.country = c.country)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@users
+POSTHOOK: Input: default@countries
+POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-23_12-31-25_489_4149172260159935049/10000
+user1 USA United States USA
+PREHOOK: query: SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c
+ON (u.country_id = c.country_id)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@users
+PREHOOK: Input: default@countries
+PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-23_12-31-32_285_599526889736525802/10000
+POSTHOOK: query: SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c
+ON (u.country_id = c.country_id)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@users
+POSTHOOK: Input: default@countries
+POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-23_12-31-32_285_599526889736525802/10000
+PREHOOK: query: SELECT u.key, u.state, s.name FROM users u JOIN states s
+ON (u.state = s.key)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@users
+PREHOOK: Input: default@states
+PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-23_12-31-38_500_2952626450057612534/10000
+POSTHOOK: query: SELECT u.key, u.state, s.name FROM users u JOIN states s
+ON (u.state = s.key)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@users
+POSTHOOK: Input: default@states
+POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-23_12-31-38_500_2952626450057612534/10000
+user1 IA Iowa
+PREHOOK: query: SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c
+ON (u.country = c.key)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@users
+PREHOOK: Input: default@countries
+PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-23_12-31-45_911_3744508358740860232/10000
+POSTHOOK: query: SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c
+ON (u.country = c.key)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@users
+POSTHOOK: Input: default@countries
+POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-23_12-31-45_911_3744508358740860232/10000
+user1 USA United States USA
+PREHOOK: query: SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c
+ON (u.country = c.country)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@users
+PREHOOK: Input: default@countries
+PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-23_12-31-52_375_497455375799515705/10000
+POSTHOOK: query: SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c
+ON (u.country = c.country)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@users
+POSTHOOK: Input: default@countries
+POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-23_12-31-52_375_497455375799515705/10000
+user1 USA United States USA
+PREHOOK: query: SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c
+ON (u.country_id = c.country_id)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@users
+PREHOOK: Input: default@countries
+PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-23_12-31-59_778_6209720937134856434/10000
+POSTHOOK: query: SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c
+ON (u.country_id = c.country_id)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@users
+POSTHOOK: Input: default@countries
+POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-23_12-31-59_778_6209720937134856434/10000
+PREHOOK: query: SELECT u.key, u.state, s.name FROM users u JOIN states s
+ON (u.state = s.key)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@users
+PREHOOK: Input: default@states
+PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-23_12-32-06_717_1044214443379190261/10000
+POSTHOOK: query: SELECT u.key, u.state, s.name FROM users u JOIN states s
+ON (u.state = s.key)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@users
+POSTHOOK: Input: default@states
+POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-23_12-32-06_717_1044214443379190261/10000
+user1 IA Iowa
+PREHOOK: query: DROP TABLE users
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE users
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: default@users
+PREHOOK: query: DROP TABLE states
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE states
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: default@states
+PREHOOK: query: DROP TABLE countries
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE countries
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: default@countries
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java?rev=926818&r1=926817&r2=926818&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java Tue Mar 23 22:29:44 2010
@@ -33,6 +33,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.plan.PartitionDesc;
+import org.apache.hadoop.hive.ql.plan.TableDesc;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.hive.shims.HadoopShims.CombineFileInputFormatShim;
import org.apache.hadoop.hive.shims.HadoopShims.InputSplitShim;
@@ -260,6 +261,11 @@ public class CombineHiveInputFormat<K ex
Class inputFormatClass = part.getInputFileFormatClass();
InputFormat inputFormat = getInputFormatFromCache(inputFormatClass, job);
+ TableDesc tableDesc = part.getTableDesc();
+ if ((tableDesc != null) && tableDesc.isNonNative()) {
+ return super.getSplits(job, numSplits);
+ }
+
if ((inputFormat instanceof TextInputFormat) &&
((new CompressionCodecFactory(job)).getCodec(tstPath) != null)) {
return super.getSplits(job, numSplits);
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java?rev=926818&r1=926817&r2=926818&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java Tue Mar 23 22:29:44 2010
@@ -220,12 +220,10 @@ public class HiveInputFormat<K extends W
InputFormat inputFormat = getInputFormatFromCache(inputFormatClass,
cloneJobConf);
- Path [] paths = FileInputFormat.getInputPaths(job);
- // for now we only get one path for splits which access a non-native
- // table; should probably add a corresponding assertion
- PartitionDesc part = getPartitionDescFromPath(
- pathToPartitionInfo, paths[0]);
- Utilities.copyTableJobPropertiesToConf(part.getTableDesc(), cloneJobConf);
+ PartitionDesc part = pathToPartitionInfo.get(hsplit.getPath().toString());
+ if ((part != null) && (part.getTableDesc() != null)) {
+ Utilities.copyTableJobPropertiesToConf(part.getTableDesc(), cloneJobConf);
+ }
return new HiveRecordReader(inputFormat.getRecordReader(inputSplit,
cloneJobConf, reporter));
}