You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pr...@apache.org on 2016/02/05 23:07:59 UTC
hive git commit: HIVE-11097: HiveInputFormat uses String.startsWith
to compare splitPath and PathToAliases (Wan Chang reviewed by Prasanth
Jayachandran)
Repository: hive
Updated Branches:
refs/heads/master 9bb785084 -> 38b172cdc
HIVE-11097: HiveInputFormat uses String.startsWith to compare splitPath and PathToAliases (Wan Chang reviewed by Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/38b172cd
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/38b172cd
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/38b172cd
Branch: refs/heads/master
Commit: 38b172cdce32fa0b9014f16b44e8dc96bdc143d3
Parents: 9bb7850
Author: Prasanth Jayachandran <j....@gmail.com>
Authored: Fri Feb 5 16:07:46 2016 -0600
Committer: Prasanth Jayachandran <j....@gmail.com>
Committed: Fri Feb 5 16:07:46 2016 -0600
----------------------------------------------------------------------
.../test/resources/testconfiguration.properties | 3 +-
.../hadoop/hive/ql/io/HiveInputFormat.java | 22 +++++-
.../queries/clientpositive/orc_mr_pathalias.q | 19 +++++
.../clientpositive/orc_mr_pathalias.q.out | 77 ++++++++++++++++++++
4 files changed, 118 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/38b172cd/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index fd2c924..b46b160 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -52,7 +52,8 @@ minimr.query.files=auto_sortmerge_join_16.q,\
temp_table_external.q,\
truncate_column_buckets.q,\
uber_reduce.q,\
- udf_using.q
+ udf_using.q,\
+ orc_mr_pathalias.q
# These tests are disabled for minimr
# ql_rewrite_gbtoidx.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/38b172cd/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
index 1c0f4cd..14a6a72 100755
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
@@ -30,6 +30,7 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.Map.Entry;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.ql.exec.SerializationUtilities;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -589,6 +590,24 @@ public class HiveInputFormat<K extends WritableComparable, V extends Writable>
splitPathWithNoSchema, false);
}
+ private static boolean isMatch(String splitPath, String key) {
+ if (splitPath.equals(key)) {
+ return true;
+ }
+ // Take care of these cases:
+ // splitPath: hdfs://ns/user/hive/warehouse/src/data.txt
+ // key: [hdfs://ns]/user/hive/warehouse/src
+ // [hdfs://ns]/user/hive/warehouse/src_2
+ // [hdfs://ns]/user/hive/warehouse/src/
+ // [hdfs://ns]/user/hive/warehouse/src/data.txt
+ key = StringUtils.removeEnd(key, "/");
+ int index = splitPath.indexOf(key);
+ if (index == -1) {
+ return false;
+ }
+ return splitPath.substring(index).equals(key) || splitPath.charAt(index+key.length()) == '/';
+ }
+
protected void pushProjectionsAndFilters(JobConf jobConf, Class inputFormatClass,
String splitPath, String splitPathWithNoSchema, boolean nonNative) {
if (this.mrwork == null) {
@@ -618,8 +637,7 @@ public class HiveInputFormat<K extends WritableComparable, V extends Writable>
// subdirectories. (Unlike non-native tables, prefix mixups don't seem
// to be a potential problem here since we are always dealing with the
// path to something deeper than the table location.)
- match =
- splitPath.startsWith(key) || splitPathWithNoSchema.startsWith(key);
+ match = isMatch(splitPath, key) || isMatch(splitPathWithNoSchema, key);
}
if (match) {
ArrayList<String> list = entry.getValue();
http://git-wip-us.apache.org/repos/asf/hive/blob/38b172cd/ql/src/test/queries/clientpositive/orc_mr_pathalias.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/orc_mr_pathalias.q b/ql/src/test/queries/clientpositive/orc_mr_pathalias.q
new file mode 100644
index 0000000..d77d07a
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/orc_mr_pathalias.q
@@ -0,0 +1,19 @@
+create database if not exists test;
+drop table if exists test.test_orc_src;
+drop table if exists test.test_orc_src2;
+create table test.test_orc_src (a int, b int, c int) stored as orc;
+create table test.test_orc_src2 (a int, b int, d int) stored as orc;
+insert overwrite table test.test_orc_src select 1,2,3 from src limit 1;
+insert overwrite table test.test_orc_src2 select 1,2,4 from src limit 1;
+
+set hive.auto.convert.join = false;
+set hive.execution.engine = mr;
+
+select
+ tb.c
+from test.test_orc_src tb
+join (
+ select * from test.test_orc_src2
+) tm
+on tb.a = tm.a
+where tb.b = 2;
http://git-wip-us.apache.org/repos/asf/hive/blob/38b172cd/ql/src/test/results/clientpositive/orc_mr_pathalias.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/orc_mr_pathalias.q.out b/ql/src/test/results/clientpositive/orc_mr_pathalias.q.out
new file mode 100644
index 0000000..ec2f6a1
--- /dev/null
+++ b/ql/src/test/results/clientpositive/orc_mr_pathalias.q.out
@@ -0,0 +1,77 @@
+PREHOOK: query: create database if not exists test
+PREHOOK: type: CREATEDATABASE
+PREHOOK: Output: database:test
+POSTHOOK: query: create database if not exists test
+POSTHOOK: type: CREATEDATABASE
+POSTHOOK: Output: database:test
+PREHOOK: query: drop table if exists test.test_orc_src
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists test.test_orc_src
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: drop table if exists test.test_orc_src2
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists test.test_orc_src2
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table test.test_orc_src (a int, b int, c int) stored as orc
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:test
+PREHOOK: Output: test@test_orc_src
+POSTHOOK: query: create table test.test_orc_src (a int, b int, c int) stored as orc
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:test
+POSTHOOK: Output: test@test_orc_src
+PREHOOK: query: create table test.test_orc_src2 (a int, b int, d int) stored as orc
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:test
+PREHOOK: Output: test@test_orc_src2
+POSTHOOK: query: create table test.test_orc_src2 (a int, b int, d int) stored as orc
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:test
+POSTHOOK: Output: test@test_orc_src2
+PREHOOK: query: insert overwrite table test.test_orc_src select 1,2,3 from src limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: test@test_orc_src
+POSTHOOK: query: insert overwrite table test.test_orc_src select 1,2,3 from src limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: test@test_orc_src
+POSTHOOK: Lineage: test_orc_src.a SIMPLE []
+POSTHOOK: Lineage: test_orc_src.b SIMPLE []
+POSTHOOK: Lineage: test_orc_src.c SIMPLE []
+PREHOOK: query: insert overwrite table test.test_orc_src2 select 1,2,4 from src limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: test@test_orc_src2
+POSTHOOK: query: insert overwrite table test.test_orc_src2 select 1,2,4 from src limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: test@test_orc_src2
+POSTHOOK: Lineage: test_orc_src2.a SIMPLE []
+POSTHOOK: Lineage: test_orc_src2.b SIMPLE []
+POSTHOOK: Lineage: test_orc_src2.d SIMPLE []
+PREHOOK: query: select
+ tb.c
+from test.test_orc_src tb
+join (
+ select * from test.test_orc_src2
+) tm
+on tb.a = tm.a
+where tb.b = 2
+PREHOOK: type: QUERY
+PREHOOK: Input: test@test_orc_src
+PREHOOK: Input: test@test_orc_src2
+#### A masked pattern was here ####
+POSTHOOK: query: select
+ tb.c
+from test.test_orc_src tb
+join (
+ select * from test.test_orc_src2
+) tm
+on tb.a = tm.a
+where tb.b = 2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: test@test_orc_src
+POSTHOOK: Input: test@test_orc_src2
+#### A masked pattern was here ####
+3