You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2015/09/22 01:03:13 UTC
[1/8] hive git commit: HIVE-11512: Hive LDAP Authenticator should
also support full DN in Authenticate() Naveen Gangam via Chaoyu Tang
Repository: hive
Updated Branches:
refs/heads/llap 7148ea090 -> 47187618b
HIVE-11512: Hive LDAP Authenticator should also support full DN in Authenticate() Naveen Gangam via Chaoyu Tang
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/cc78dd5d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/cc78dd5d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/cc78dd5d
Branch: refs/heads/llap
Commit: cc78dd5d8ce1e5a77ecf9f65b9bd19cccf158ac3
Parents: ae588db
Author: ctang <ct...@gmail.com>
Authored: Fri Sep 18 22:02:22 2015 -0400
Committer: ctang <ct...@gmail.com>
Committed: Fri Sep 18 22:03:42 2015 -0400
----------------------------------------------------------------------
.../auth/LdapAuthenticationProviderImpl.java | 82 ++++++++++++++++++--
1 file changed, 76 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/cc78dd5d/service/src/java/org/apache/hive/service/auth/LdapAuthenticationProviderImpl.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/auth/LdapAuthenticationProviderImpl.java b/service/src/java/org/apache/hive/service/auth/LdapAuthenticationProviderImpl.java
index 0c7cede..b2c4daf 100644
--- a/service/src/java/org/apache/hive/service/auth/LdapAuthenticationProviderImpl.java
+++ b/service/src/java/org/apache/hive/service/auth/LdapAuthenticationProviderImpl.java
@@ -146,15 +146,28 @@ public class LdapAuthenticationProviderImpl implements PasswdAuthenticationProvi
DirContext ctx = null;
String userDN = null;
+ String userName = null;
try {
// Create initial context
ctx = new InitialDirContext(env);
+ if (isDN(user)) {
+ userName = extractName(user);
+ } else {
+ userName = user;
+ }
+
if (userFilter == null && groupFilter == null && customQuery == null) {
- userDN = findUserDNByPattern(ctx, user);
+ if (isDN(user)) {
+ userDN = findUserDNByDN(ctx, user);
+ } else {
+ if (userDN == null) {
+ userDN = findUserDNByPattern(ctx, user);
+ }
- if (userDN == null) {
- userDN = findUserDNByName(ctx, baseDN, user);
+ if (userDN == null) {
+ userDN = findUserDNByName(ctx, baseDN, user);
+ }
}
// This should not be null because we were allowed to bind with this username
@@ -185,7 +198,7 @@ public class LdapAuthenticationProviderImpl implements PasswdAuthenticationProvi
boolean success = false;
for (String filteredUser : userFilter) {
- if (filteredUser.equalsIgnoreCase(user)) {
+ if (filteredUser.equalsIgnoreCase(userName)) {
LOG.debug("User filter partially satisfied");
success = true;
break;
@@ -198,7 +211,7 @@ public class LdapAuthenticationProviderImpl implements PasswdAuthenticationProvi
"of specified list");
}
- userDN = findUserDNByPattern(ctx, user);
+ userDN = findUserDNByPattern(ctx, userName);
if (userDN != null) {
LOG.info("User filter entirely satisfied");
} else {
@@ -214,7 +227,7 @@ public class LdapAuthenticationProviderImpl implements PasswdAuthenticationProvi
// if only groupFilter is configured.
if (userDN == null) {
- userDN = findUserDNByName(ctx, baseDN, user);
+ userDN = findUserDNByName(ctx, baseDN, userName);
}
List<String> userGroups = getGroupsForUser(ctx, userDN);
@@ -395,6 +408,44 @@ public class LdapAuthenticationProviderImpl implements PasswdAuthenticationProvi
return null;
}
+ /**
+ * This helper method attempts to find a username given a DN.
+ * Various LDAP implementations have different keys/properties that store this unique userID.
+ * Active Directory has a "sAMAccountName" that appears reliable,openLDAP uses "uid"
+ * So the first attempt is to find an entity with objectClass=person||user where
+ * (uid||sAMAccountName) matches the given username.
+ * The second attempt is to use CN attribute for wild card matching and then match the
+ * username in the DN.
+ * @param ctx DirContext for the LDAP Connection.
+ * @param baseDN BaseDN for this LDAP directory where the search is to be performed.
+ * @param userName A unique userid that is to be located in the LDAP.
+ * @return LDAP DN if the user is found in LDAP, null otherwise.
+ */
+ public static String findUserDNByDN(DirContext ctx, String userDN)
+ throws NamingException {
+ if (!isDN(userDN)) {
+ return null;
+ }
+
+ String baseDN = extractBaseDN(userDN);
+ List<String> results = null;
+ String searchFilter = "(&(|(objectClass=person)(objectClass=user))(" + DN_ATTR + "="
+ + userDN + "))";
+
+ results = findDNByName(ctx, baseDN, searchFilter, 2);
+
+ if (results == null) {
+ return null;
+ }
+
+ if(results.size() > 1) {
+ //make sure there is not another item available, there should be only 1 match
+ LOG.info("Matched multiple users for the user: " + userDN + ",returning null");
+ return null;
+ }
+ return userDN;
+ }
+
public static List<String> findDNByName(DirContext ctx, String baseDN,
String searchString, int limit) throws NamingException {
SearchResult searchResult = null;
@@ -507,4 +558,23 @@ public class LdapAuthenticationProviderImpl implements PasswdAuthenticationProvi
}
return list;
}
+
+ public static boolean isDN(String name) {
+ return (name.indexOf("=") > -1);
+ }
+
+ public static String extractName(String dn) {
+ if (dn.indexOf("=") > -1) {
+ return dn.substring(dn.indexOf("=") + 1, dn.indexOf(","));
+ }
+ return dn;
+ }
+
+ public static String extractBaseDN(String dn) {
+ if (dn.indexOf(",") > -1) {
+ return dn.substring(dn.indexOf(",") + 1);
+ }
+ return null;
+ }
+
}
[4/8] hive git commit: HIVE-11843: Add 'sort by c' to Parquet PPD
q-tests to avoid different output issues with hadoop-1 (Sergio Pena,
reviewed by Ferdinand Xu)
Posted by se...@apache.org.
HIVE-11843: Add 'sort by c' to Parquet PPD q-tests to avoid different output issues with hadoop-1 (Sergio Pena, reviewed by Ferdinand Xu)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/92b42ae9
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/92b42ae9
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/92b42ae9
Branch: refs/heads/llap
Commit: 92b42ae9efaa2da352c123a0bb74adf2d3ea267d
Parents: 2186159
Author: Sergio Pena <se...@cloudera.com>
Authored: Mon Sep 21 10:06:10 2015 -0500
Committer: Sergio Pena <se...@cloudera.com>
Committed: Mon Sep 21 10:06:10 2015 -0500
----------------------------------------------------------------------
.../clientpositive/parquet_ppd_boolean.q | 4 +-
.../queries/clientpositive/parquet_ppd_char.q | 12 +-
.../queries/clientpositive/parquet_ppd_date.q | 16 +-
.../clientpositive/parquet_ppd_decimal.q | 32 +--
.../clientpositive/parquet_ppd_timestamp.q | 16 +-
.../clientpositive/parquet_ppd_varchar.q | 12 +-
.../clientpositive/parquet_ppd_boolean.q.out | 28 +--
.../clientpositive/parquet_ppd_char.q.out | 84 +++----
.../clientpositive/parquet_ppd_date.q.out | 112 +++++-----
.../clientpositive/parquet_ppd_decimal.q.out | 224 +++++++++----------
.../clientpositive/parquet_ppd_timestamp.q.out | 112 +++++-----
.../clientpositive/parquet_ppd_varchar.q.out | 84 +++----
12 files changed, 368 insertions(+), 368 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/92b42ae9/ql/src/test/queries/clientpositive/parquet_ppd_boolean.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/parquet_ppd_boolean.q b/ql/src/test/queries/clientpositive/parquet_ppd_boolean.q
index a7848b4..059da68 100644
--- a/ql/src/test/queries/clientpositive/parquet_ppd_boolean.q
+++ b/ql/src/test/queries/clientpositive/parquet_ppd_boolean.q
@@ -12,7 +12,7 @@ select * from newtypestbl where b=true;
select * from newtypestbl where b!=true;
select * from newtypestbl where b<true;
select * from newtypestbl where b>true;
-select * from newtypestbl where b<=true;
+select * from newtypestbl where b<=true sort by c;
select * from newtypestbl where b=false;
select * from newtypestbl where b!=false;
@@ -26,7 +26,7 @@ select * from newtypestbl where b=true;
select * from newtypestbl where b!=true;
select * from newtypestbl where b<true;
select * from newtypestbl where b>true;
-select * from newtypestbl where b<=true;
+select * from newtypestbl where b<=true sort by c;
select * from newtypestbl where b=false;
select * from newtypestbl where b!=false;
http://git-wip-us.apache.org/repos/asf/hive/blob/92b42ae9/ql/src/test/queries/clientpositive/parquet_ppd_char.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/parquet_ppd_char.q b/ql/src/test/queries/clientpositive/parquet_ppd_char.q
index dcad622..eaddcb4 100644
--- a/ql/src/test/queries/clientpositive/parquet_ppd_char.q
+++ b/ql/src/test/queries/clientpositive/parquet_ppd_char.q
@@ -28,10 +28,10 @@ set hive.optimize.index.filter=true;
select * from newtypestbl where c<"hello";
set hive.optimize.index.filter=false;
-select * from newtypestbl where c<="hello";
+select * from newtypestbl where c<="hello" sort by c;
set hive.optimize.index.filter=true;
-select * from newtypestbl where c<="hello";
+select * from newtypestbl where c<="hello" sort by c;
set hive.optimize.index.filter=false;
select * from newtypestbl where c="apple ";
@@ -46,10 +46,10 @@ set hive.optimize.index.filter=true;
select * from newtypestbl where c in ("apple", "carrot");
set hive.optimize.index.filter=false;
-select * from newtypestbl where c in ("apple", "hello");
+select * from newtypestbl where c in ("apple", "hello") sort by c;
set hive.optimize.index.filter=true;
-select * from newtypestbl where c in ("apple", "hello");
+select * from newtypestbl where c in ("apple", "hello") sort by c;
set hive.optimize.index.filter=false;
select * from newtypestbl where c in ("carrot");
@@ -64,10 +64,10 @@ set hive.optimize.index.filter=true;
select * from newtypestbl where c between "apple" and "carrot";
set hive.optimize.index.filter=false;
-select * from newtypestbl where c between "apple" and "zombie";
+select * from newtypestbl where c between "apple" and "zombie" sort by c;
set hive.optimize.index.filter=true;
-select * from newtypestbl where c between "apple" and "zombie";
+select * from newtypestbl where c between "apple" and "zombie" sort by c;
set hive.optimize.index.filter=false;
select * from newtypestbl where c between "carrot" and "carrot1";
http://git-wip-us.apache.org/repos/asf/hive/blob/92b42ae9/ql/src/test/queries/clientpositive/parquet_ppd_date.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/parquet_ppd_date.q b/ql/src/test/queries/clientpositive/parquet_ppd_date.q
index a05d358..41d0d64 100644
--- a/ql/src/test/queries/clientpositive/parquet_ppd_date.q
+++ b/ql/src/test/queries/clientpositive/parquet_ppd_date.q
@@ -41,10 +41,10 @@ set hive.optimize.index.filter=true;
select * from newtypestbl where da<'1970-02-27';
set hive.optimize.index.filter=false;
-select * from newtypestbl where da<'1970-02-29';
+select * from newtypestbl where da<'1970-02-29' sort by c;
set hive.optimize.index.filter=true;
-select * from newtypestbl where da<'1970-02-29';
+select * from newtypestbl where da<'1970-02-29' sort by c;
set hive.optimize.index.filter=false;
select * from newtypestbl where da<'1970-02-15';
@@ -59,10 +59,10 @@ set hive.optimize.index.filter=true;
select * from newtypestbl where da<='1970-02-20';
set hive.optimize.index.filter=false;
-select * from newtypestbl where da<='1970-02-27';
+select * from newtypestbl where da<='1970-02-27' sort by c;
set hive.optimize.index.filter=true;
-select * from newtypestbl where da<='1970-02-27';
+select * from newtypestbl where da<='1970-02-27' sort by c;
set hive.optimize.index.filter=false;
select * from newtypestbl where da in (cast('1970-02-21' as date), cast('1970-02-27' as date));
@@ -71,10 +71,10 @@ set hive.optimize.index.filter=true;
select * from newtypestbl where da in (cast('1970-02-21' as date), cast('1970-02-27' as date));
set hive.optimize.index.filter=false;
-select * from newtypestbl where da in (cast('1970-02-20' as date), cast('1970-02-27' as date));
+select * from newtypestbl where da in (cast('1970-02-20' as date), cast('1970-02-27' as date)) sort by c;
set hive.optimize.index.filter=true;
-select * from newtypestbl where da in (cast('1970-02-20' as date), cast('1970-02-27' as date));
+select * from newtypestbl where da in (cast('1970-02-20' as date), cast('1970-02-27' as date)) sort by c;
set hive.optimize.index.filter=false;
select * from newtypestbl where da in (cast('1970-02-21' as date), cast('1970-02-22' as date));
@@ -89,10 +89,10 @@ set hive.optimize.index.filter=true;
select * from newtypestbl where da between '1970-02-19' and '1970-02-22';
set hive.optimize.index.filter=false;
-select * from newtypestbl where da between '1970-02-19' and '1970-02-28';
+select * from newtypestbl where da between '1970-02-19' and '1970-02-28' sort by c;
set hive.optimize.index.filter=true;
-select * from newtypestbl where da between '1970-02-19' and '1970-02-28';
+select * from newtypestbl where da between '1970-02-19' and '1970-02-28' sort by c;
set hive.optimize.index.filter=false;
select * from newtypestbl where da between '1970-02-18' and '1970-02-19';
http://git-wip-us.apache.org/repos/asf/hive/blob/92b42ae9/ql/src/test/queries/clientpositive/parquet_ppd_decimal.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/parquet_ppd_decimal.q b/ql/src/test/queries/clientpositive/parquet_ppd_decimal.q
index cf7cba0..dfca486 100644
--- a/ql/src/test/queries/clientpositive/parquet_ppd_decimal.q
+++ b/ql/src/test/queries/clientpositive/parquet_ppd_decimal.q
@@ -67,22 +67,22 @@ set hive.optimize.index.filter=true;
select * from newtypestbl where d<1;
set hive.optimize.index.filter=false;
-select * from newtypestbl where d<=11.22;
+select * from newtypestbl where d<=11.22 sort by c;
set hive.optimize.index.filter=true;
-select * from newtypestbl where d<=11.22;
+select * from newtypestbl where d<=11.22 sort by c;
set hive.optimize.index.filter=false;
-select * from newtypestbl where d<='11.22';
+select * from newtypestbl where d<='11.22' sort by c;
set hive.optimize.index.filter=true;
-select * from newtypestbl where d<='11.22';
+select * from newtypestbl where d<='11.22' sort by c;
set hive.optimize.index.filter=false;
-select * from newtypestbl where d<=cast('11.22' as float);
+select * from newtypestbl where d<=cast('11.22' as float) sort by c;
set hive.optimize.index.filter=true;
-select * from newtypestbl where d<=cast('11.22' as float);
+select * from newtypestbl where d<=cast('11.22' as float) sort by c;
set hive.optimize.index.filter=false;
select * from newtypestbl where d<=cast('11.22' as decimal);
@@ -91,16 +91,16 @@ set hive.optimize.index.filter=true;
select * from newtypestbl where d<=cast('11.22' as decimal);
set hive.optimize.index.filter=false;
-select * from newtypestbl where d<=11.22BD;
+select * from newtypestbl where d<=11.22BD sort by c;
set hive.optimize.index.filter=true;
-select * from newtypestbl where d<=11.22BD;
+select * from newtypestbl where d<=11.22BD sort by c;
set hive.optimize.index.filter=false;
-select * from newtypestbl where d<=12;
+select * from newtypestbl where d<=12 sort by c;
set hive.optimize.index.filter=true;
-select * from newtypestbl where d<=12;
+select * from newtypestbl where d<=12 sort by c;
set hive.optimize.index.filter=false;
select * from newtypestbl where d in ('0.22', '1.0');
@@ -109,10 +109,10 @@ set hive.optimize.index.filter=true;
select * from newtypestbl where d in ('0.22', '1.0');
set hive.optimize.index.filter=false;
-select * from newtypestbl where d in ('0.22', '11.22');
+select * from newtypestbl where d in ('0.22', '11.22') sort by c;
set hive.optimize.index.filter=true;
-select * from newtypestbl where d in ('0.22', '11.22');
+select * from newtypestbl where d in ('0.22', '11.22') sort by c;
set hive.optimize.index.filter=false;
select * from newtypestbl where d in ('0.9', '1.0');
@@ -127,10 +127,10 @@ set hive.optimize.index.filter=true;
select * from newtypestbl where d in ('0.9', 0.22);
set hive.optimize.index.filter=false;
-select * from newtypestbl where d in ('0.9', 0.22, cast('11.22' as float));
+select * from newtypestbl where d in ('0.9', 0.22, cast('11.22' as float)) sort by c;
set hive.optimize.index.filter=true;
-select * from newtypestbl where d in ('0.9', 0.22, cast('11.22' as float));
+select * from newtypestbl where d in ('0.9', 0.22, cast('11.22' as float)) sort by c;
set hive.optimize.index.filter=false;
select * from newtypestbl where d between 0 and 1;
@@ -139,10 +139,10 @@ set hive.optimize.index.filter=true;
select * from newtypestbl where d between 0 and 1;
set hive.optimize.index.filter=false;
-select * from newtypestbl where d between 0 and 1000;
+select * from newtypestbl where d between 0 and 1000 sort by c;
set hive.optimize.index.filter=true;
-select * from newtypestbl where d between 0 and 1000;
+select * from newtypestbl where d between 0 and 1000 sort by c;
set hive.optimize.index.filter=false;
select * from newtypestbl where d between 0 and '2.0';
http://git-wip-us.apache.org/repos/asf/hive/blob/92b42ae9/ql/src/test/queries/clientpositive/parquet_ppd_timestamp.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/parquet_ppd_timestamp.q b/ql/src/test/queries/clientpositive/parquet_ppd_timestamp.q
index 6ed1e55..1b9f6ff 100644
--- a/ql/src/test/queries/clientpositive/parquet_ppd_timestamp.q
+++ b/ql/src/test/queries/clientpositive/parquet_ppd_timestamp.q
@@ -38,10 +38,10 @@ set hive.optimize.index.filter=true;
select * from newtypestbl where ts<cast('2011-01-20 01:01:01' as timestamp);
set hive.optimize.index.filter=false;
-select * from newtypestbl where ts<cast('2011-01-22 01:01:01' as timestamp);
+select * from newtypestbl where ts<cast('2011-01-22 01:01:01' as timestamp) sort by c;
set hive.optimize.index.filter=true;
-select * from newtypestbl where ts<cast('2011-01-22 01:01:01' as timestamp);
+select * from newtypestbl where ts<cast('2011-01-22 01:01:01' as timestamp) sort by c;
set hive.optimize.index.filter=false;
select * from newtypestbl where ts<cast('2010-10-01 01:01:01' as timestamp);
@@ -56,10 +56,10 @@ set hive.optimize.index.filter=true;
select * from newtypestbl where ts<=cast('2011-01-01 01:01:01' as timestamp);
set hive.optimize.index.filter=false;
-select * from newtypestbl where ts<=cast('2011-01-20 01:01:01' as timestamp);
+select * from newtypestbl where ts<=cast('2011-01-20 01:01:01' as timestamp) sort by c;
set hive.optimize.index.filter=true;
-select * from newtypestbl where ts<=cast('2011-01-20 01:01:01' as timestamp);
+select * from newtypestbl where ts<=cast('2011-01-20 01:01:01' as timestamp) sort by c;
set hive.optimize.index.filter=false;
select * from newtypestbl where ts in (cast('2011-01-02 01:01:01' as timestamp), cast('2011-01-20 01:01:01' as timestamp));
@@ -68,10 +68,10 @@ set hive.optimize.index.filter=true;
select * from newtypestbl where ts in (cast('2011-01-02 01:01:01' as timestamp), cast('2011-01-20 01:01:01' as timestamp));
set hive.optimize.index.filter=false;
-select * from newtypestbl where ts in (cast('2011-01-01 01:01:01' as timestamp), cast('2011-01-20 01:01:01' as timestamp));
+select * from newtypestbl where ts in (cast('2011-01-01 01:01:01' as timestamp), cast('2011-01-20 01:01:01' as timestamp)) sort by c;
set hive.optimize.index.filter=true;
-select * from newtypestbl where ts in (cast('2011-01-01 01:01:01' as timestamp), cast('2011-01-20 01:01:01' as timestamp));
+select * from newtypestbl where ts in (cast('2011-01-01 01:01:01' as timestamp), cast('2011-01-20 01:01:01' as timestamp)) sort by c;
set hive.optimize.index.filter=false;
select * from newtypestbl where ts in (cast('2011-01-02 01:01:01' as timestamp), cast('2011-01-08 01:01:01' as timestamp));
@@ -86,10 +86,10 @@ set hive.optimize.index.filter=true;
select * from newtypestbl where ts between cast('2010-10-01 01:01:01' as timestamp) and cast('2011-01-08 01:01:01' as timestamp);
set hive.optimize.index.filter=false;
-select * from newtypestbl where ts between cast('2010-10-01 01:01:01' as timestamp) and cast('2011-01-25 01:01:01' as timestamp);
+select * from newtypestbl where ts between cast('2010-10-01 01:01:01' as timestamp) and cast('2011-01-25 01:01:01' as timestamp) sort by c;
set hive.optimize.index.filter=true;
-select * from newtypestbl where ts between cast('2010-10-01 01:01:01' as timestamp) and cast('2011-01-25 01:01:01' as timestamp);
+select * from newtypestbl where ts between cast('2010-10-01 01:01:01' as timestamp) and cast('2011-01-25 01:01:01' as timestamp) sort by c;
set hive.optimize.index.filter=false;
select * from newtypestbl where ts between cast('2010-10-01 01:01:01' as timestamp) and cast('2010-11-01 01:01:01' as timestamp);
http://git-wip-us.apache.org/repos/asf/hive/blob/92b42ae9/ql/src/test/queries/clientpositive/parquet_ppd_varchar.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/parquet_ppd_varchar.q b/ql/src/test/queries/clientpositive/parquet_ppd_varchar.q
index 41bf7df..6449c6d 100644
--- a/ql/src/test/queries/clientpositive/parquet_ppd_varchar.q
+++ b/ql/src/test/queries/clientpositive/parquet_ppd_varchar.q
@@ -28,10 +28,10 @@ set hive.optimize.index.filter=true;
select * from newtypestbl where v<"world";
set hive.optimize.index.filter=false;
-select * from newtypestbl where v<="world";
+select * from newtypestbl where v<="world" sort by c;
set hive.optimize.index.filter=true;
-select * from newtypestbl where v<="world";
+select * from newtypestbl where v<="world" sort by c;
set hive.optimize.index.filter=false;
select * from newtypestbl where v="bee ";
@@ -46,10 +46,10 @@ set hive.optimize.index.filter=true;
select * from newtypestbl where v in ("bee", "orange");
set hive.optimize.index.filter=false;
-select * from newtypestbl where v in ("bee", "world");
+select * from newtypestbl where v in ("bee", "world") sort by c;
set hive.optimize.index.filter=true;
-select * from newtypestbl where v in ("bee", "world");
+select * from newtypestbl where v in ("bee", "world") sort by c;
set hive.optimize.index.filter=false;
select * from newtypestbl where v in ("orange");
@@ -64,10 +64,10 @@ set hive.optimize.index.filter=true;
select * from newtypestbl where v between "bee" and "orange";
set hive.optimize.index.filter=false;
-select * from newtypestbl where v between "bee" and "zombie";
+select * from newtypestbl where v between "bee" and "zombie" sort by c;
set hive.optimize.index.filter=true;
-select * from newtypestbl where v between "bee" and "zombie";
+select * from newtypestbl where v between "bee" and "zombie" sort by c;
set hive.optimize.index.filter=false;
select * from newtypestbl where v between "orange" and "pine";
http://git-wip-us.apache.org/repos/asf/hive/blob/92b42ae9/ql/src/test/results/clientpositive/parquet_ppd_boolean.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/parquet_ppd_boolean.q.out b/ql/src/test/results/clientpositive/parquet_ppd_boolean.q.out
index 51ea879..1355849 100644
--- a/ql/src/test/results/clientpositive/parquet_ppd_boolean.q.out
+++ b/ql/src/test/results/clientpositive/parquet_ppd_boolean.q.out
@@ -65,24 +65,24 @@ POSTHOOK: query: select * from newtypestbl where b>true
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-PREHOOK: query: select * from newtypestbl where b<=true
+PREHOOK: query: select * from newtypestbl where b<=true sort by c
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-POSTHOOK: query: select * from newtypestbl where b<=true
+POSTHOOK: query: select * from newtypestbl where b<=true sort by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-hello world 11.22 false
apple bee 0.22 true
-hello world 11.22 false
apple bee 0.22 true
-hello world 11.22 false
apple bee 0.22 true
-hello world 11.22 false
apple bee 0.22 true
-hello world 11.22 false
apple bee 0.22 true
+hello world 11.22 false
+hello world 11.22 false
+hello world 11.22 false
+hello world 11.22 false
+hello world 11.22 false
PREHOOK: query: select * from newtypestbl where b=false
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
@@ -190,24 +190,24 @@ POSTHOOK: query: select * from newtypestbl where b>true
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-PREHOOK: query: select * from newtypestbl where b<=true
+PREHOOK: query: select * from newtypestbl where b<=true sort by c
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-POSTHOOK: query: select * from newtypestbl where b<=true
+POSTHOOK: query: select * from newtypestbl where b<=true sort by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-hello world 11.22 false
apple bee 0.22 true
-hello world 11.22 false
apple bee 0.22 true
-hello world 11.22 false
apple bee 0.22 true
-hello world 11.22 false
apple bee 0.22 true
-hello world 11.22 false
apple bee 0.22 true
+hello world 11.22 false
+hello world 11.22 false
+hello world 11.22 false
+hello world 11.22 false
+hello world 11.22 false
PREHOOK: query: select * from newtypestbl where b=false
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
http://git-wip-us.apache.org/repos/asf/hive/blob/92b42ae9/ql/src/test/results/clientpositive/parquet_ppd_char.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/parquet_ppd_char.q.out b/ql/src/test/results/clientpositive/parquet_ppd_char.q.out
index af4a13c..f224870 100644
--- a/ql/src/test/results/clientpositive/parquet_ppd_char.q.out
+++ b/ql/src/test/results/clientpositive/parquet_ppd_char.q.out
@@ -98,42 +98,42 @@ apple bee 0.22 1970-02-20
apple bee 0.22 1970-02-20
apple bee 0.22 1970-02-20
apple bee 0.22 1970-02-20
-PREHOOK: query: select * from newtypestbl where c<="hello"
+PREHOOK: query: select * from newtypestbl where c<="hello" sort by c
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-POSTHOOK: query: select * from newtypestbl where c<="hello"
+POSTHOOK: query: select * from newtypestbl where c<="hello" sort by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-PREHOOK: query: select * from newtypestbl where c<="hello"
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+PREHOOK: query: select * from newtypestbl where c<="hello" sort by c
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-POSTHOOK: query: select * from newtypestbl where c<="hello"
+POSTHOOK: query: select * from newtypestbl where c<="hello" sort by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
PREHOOK: query: select * from newtypestbl where c="apple "
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
@@ -176,42 +176,42 @@ apple bee 0.22 1970-02-20
apple bee 0.22 1970-02-20
apple bee 0.22 1970-02-20
apple bee 0.22 1970-02-20
-PREHOOK: query: select * from newtypestbl where c in ("apple", "hello")
+PREHOOK: query: select * from newtypestbl where c in ("apple", "hello") sort by c
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-POSTHOOK: query: select * from newtypestbl where c in ("apple", "hello")
+POSTHOOK: query: select * from newtypestbl where c in ("apple", "hello") sort by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-PREHOOK: query: select * from newtypestbl where c in ("apple", "hello")
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+PREHOOK: query: select * from newtypestbl where c in ("apple", "hello") sort by c
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-POSTHOOK: query: select * from newtypestbl where c in ("apple", "hello")
+POSTHOOK: query: select * from newtypestbl where c in ("apple", "hello") sort by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
PREHOOK: query: select * from newtypestbl where c in ("carrot")
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
@@ -254,42 +254,42 @@ apple bee 0.22 1970-02-20
apple bee 0.22 1970-02-20
apple bee 0.22 1970-02-20
apple bee 0.22 1970-02-20
-PREHOOK: query: select * from newtypestbl where c between "apple" and "zombie"
+PREHOOK: query: select * from newtypestbl where c between "apple" and "zombie" sort by c
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-POSTHOOK: query: select * from newtypestbl where c between "apple" and "zombie"
+POSTHOOK: query: select * from newtypestbl where c between "apple" and "zombie" sort by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-PREHOOK: query: select * from newtypestbl where c between "apple" and "zombie"
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+PREHOOK: query: select * from newtypestbl where c between "apple" and "zombie" sort by c
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-POSTHOOK: query: select * from newtypestbl where c between "apple" and "zombie"
+POSTHOOK: query: select * from newtypestbl where c between "apple" and "zombie" sort by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
PREHOOK: query: select * from newtypestbl where c between "carrot" and "carrot1"
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
http://git-wip-us.apache.org/repos/asf/hive/blob/92b42ae9/ql/src/test/results/clientpositive/parquet_ppd_date.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/parquet_ppd_date.q.out b/ql/src/test/results/clientpositive/parquet_ppd_date.q.out
index 60c9a59..e599014 100644
--- a/ql/src/test/results/clientpositive/parquet_ppd_date.q.out
+++ b/ql/src/test/results/clientpositive/parquet_ppd_date.q.out
@@ -163,42 +163,42 @@ apple bee 0.22 1970-02-20
apple bee 0.22 1970-02-20
apple bee 0.22 1970-02-20
apple bee 0.22 1970-02-20
-PREHOOK: query: select * from newtypestbl where da<'1970-02-29'
+PREHOOK: query: select * from newtypestbl where da<'1970-02-29' sort by c
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-POSTHOOK: query: select * from newtypestbl where da<'1970-02-29'
+POSTHOOK: query: select * from newtypestbl where da<'1970-02-29' sort by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-PREHOOK: query: select * from newtypestbl where da<'1970-02-29'
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+PREHOOK: query: select * from newtypestbl where da<'1970-02-29' sort by c
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-POSTHOOK: query: select * from newtypestbl where da<'1970-02-29'
+POSTHOOK: query: select * from newtypestbl where da<'1970-02-29' sort by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
PREHOOK: query: select * from newtypestbl where da<'1970-02-15'
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
@@ -241,42 +241,42 @@ apple bee 0.22 1970-02-20
apple bee 0.22 1970-02-20
apple bee 0.22 1970-02-20
apple bee 0.22 1970-02-20
-PREHOOK: query: select * from newtypestbl where da<='1970-02-27'
+PREHOOK: query: select * from newtypestbl where da<='1970-02-27' sort by c
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-POSTHOOK: query: select * from newtypestbl where da<='1970-02-27'
+POSTHOOK: query: select * from newtypestbl where da<='1970-02-27' sort by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-PREHOOK: query: select * from newtypestbl where da<='1970-02-27'
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+PREHOOK: query: select * from newtypestbl where da<='1970-02-27' sort by c
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-POSTHOOK: query: select * from newtypestbl where da<='1970-02-27'
+POSTHOOK: query: select * from newtypestbl where da<='1970-02-27' sort by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
PREHOOK: query: select * from newtypestbl where da in (cast('1970-02-21' as date), cast('1970-02-27' as date))
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
@@ -303,42 +303,42 @@ hello world 11.22 1970-02-27
hello world 11.22 1970-02-27
hello world 11.22 1970-02-27
hello world 11.22 1970-02-27
-PREHOOK: query: select * from newtypestbl where da in (cast('1970-02-20' as date), cast('1970-02-27' as date))
+PREHOOK: query: select * from newtypestbl where da in (cast('1970-02-20' as date), cast('1970-02-27' as date)) sort by c
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-POSTHOOK: query: select * from newtypestbl where da in (cast('1970-02-20' as date), cast('1970-02-27' as date))
+POSTHOOK: query: select * from newtypestbl where da in (cast('1970-02-20' as date), cast('1970-02-27' as date)) sort by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-PREHOOK: query: select * from newtypestbl where da in (cast('1970-02-20' as date), cast('1970-02-27' as date))
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+PREHOOK: query: select * from newtypestbl where da in (cast('1970-02-20' as date), cast('1970-02-27' as date)) sort by c
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-POSTHOOK: query: select * from newtypestbl where da in (cast('1970-02-20' as date), cast('1970-02-27' as date))
+POSTHOOK: query: select * from newtypestbl where da in (cast('1970-02-20' as date), cast('1970-02-27' as date)) sort by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
PREHOOK: query: select * from newtypestbl where da in (cast('1970-02-21' as date), cast('1970-02-22' as date))
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
@@ -381,42 +381,42 @@ apple bee 0.22 1970-02-20
apple bee 0.22 1970-02-20
apple bee 0.22 1970-02-20
apple bee 0.22 1970-02-20
-PREHOOK: query: select * from newtypestbl where da between '1970-02-19' and '1970-02-28'
+PREHOOK: query: select * from newtypestbl where da between '1970-02-19' and '1970-02-28' sort by c
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-POSTHOOK: query: select * from newtypestbl where da between '1970-02-19' and '1970-02-28'
+POSTHOOK: query: select * from newtypestbl where da between '1970-02-19' and '1970-02-28' sort by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-PREHOOK: query: select * from newtypestbl where da between '1970-02-19' and '1970-02-28'
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+PREHOOK: query: select * from newtypestbl where da between '1970-02-19' and '1970-02-28' sort by c
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-POSTHOOK: query: select * from newtypestbl where da between '1970-02-19' and '1970-02-28'
+POSTHOOK: query: select * from newtypestbl where da between '1970-02-19' and '1970-02-28' sort by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
PREHOOK: query: select * from newtypestbl where da between '1970-02-18' and '1970-02-19'
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
http://git-wip-us.apache.org/repos/asf/hive/blob/92b42ae9/ql/src/test/results/clientpositive/parquet_ppd_decimal.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/parquet_ppd_decimal.q.out b/ql/src/test/results/clientpositive/parquet_ppd_decimal.q.out
index ec603eb..7c17733 100644
--- a/ql/src/test/results/clientpositive/parquet_ppd_decimal.q.out
+++ b/ql/src/test/results/clientpositive/parquet_ppd_decimal.q.out
@@ -280,114 +280,114 @@ apple bee 0.22 1970-02-20
apple bee 0.22 1970-02-20
apple bee 0.22 1970-02-20
apple bee 0.22 1970-02-20
-PREHOOK: query: select * from newtypestbl where d<=11.22
+PREHOOK: query: select * from newtypestbl where d<=11.22 sort by c
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-POSTHOOK: query: select * from newtypestbl where d<=11.22
+POSTHOOK: query: select * from newtypestbl where d<=11.22 sort by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-PREHOOK: query: select * from newtypestbl where d<=11.22
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+PREHOOK: query: select * from newtypestbl where d<=11.22 sort by c
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-POSTHOOK: query: select * from newtypestbl where d<=11.22
+POSTHOOK: query: select * from newtypestbl where d<=11.22 sort by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-PREHOOK: query: select * from newtypestbl where d<='11.22'
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+PREHOOK: query: select * from newtypestbl where d<='11.22' sort by c
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-POSTHOOK: query: select * from newtypestbl where d<='11.22'
+POSTHOOK: query: select * from newtypestbl where d<='11.22' sort by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-PREHOOK: query: select * from newtypestbl where d<='11.22'
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+PREHOOK: query: select * from newtypestbl where d<='11.22' sort by c
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-POSTHOOK: query: select * from newtypestbl where d<='11.22'
+POSTHOOK: query: select * from newtypestbl where d<='11.22' sort by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-PREHOOK: query: select * from newtypestbl where d<=cast('11.22' as float)
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+PREHOOK: query: select * from newtypestbl where d<=cast('11.22' as float) sort by c
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-POSTHOOK: query: select * from newtypestbl where d<=cast('11.22' as float)
+POSTHOOK: query: select * from newtypestbl where d<=cast('11.22' as float) sort by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-PREHOOK: query: select * from newtypestbl where d<=cast('11.22' as float)
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+PREHOOK: query: select * from newtypestbl where d<=cast('11.22' as float) sort by c
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-POSTHOOK: query: select * from newtypestbl where d<=cast('11.22' as float)
+POSTHOOK: query: select * from newtypestbl where d<=cast('11.22' as float) sort by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
PREHOOK: query: select * from newtypestbl where d<=cast('11.22' as decimal)
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
@@ -414,78 +414,78 @@ apple bee 0.22 1970-02-20
apple bee 0.22 1970-02-20
apple bee 0.22 1970-02-20
apple bee 0.22 1970-02-20
-PREHOOK: query: select * from newtypestbl where d<=11.22BD
+PREHOOK: query: select * from newtypestbl where d<=11.22BD sort by c
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-POSTHOOK: query: select * from newtypestbl where d<=11.22BD
+POSTHOOK: query: select * from newtypestbl where d<=11.22BD sort by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-PREHOOK: query: select * from newtypestbl where d<=11.22BD
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+PREHOOK: query: select * from newtypestbl where d<=11.22BD sort by c
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-POSTHOOK: query: select * from newtypestbl where d<=11.22BD
+POSTHOOK: query: select * from newtypestbl where d<=11.22BD sort by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-PREHOOK: query: select * from newtypestbl where d<=12
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+PREHOOK: query: select * from newtypestbl where d<=12 sort by c
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-POSTHOOK: query: select * from newtypestbl where d<=12
+POSTHOOK: query: select * from newtypestbl where d<=12 sort by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-PREHOOK: query: select * from newtypestbl where d<=12
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+PREHOOK: query: select * from newtypestbl where d<=12 sort by c
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-POSTHOOK: query: select * from newtypestbl where d<=12
+POSTHOOK: query: select * from newtypestbl where d<=12 sort by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
PREHOOK: query: select * from newtypestbl where d in ('0.22', '1.0')
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
@@ -512,42 +512,42 @@ apple bee 0.22 1970-02-20
apple bee 0.22 1970-02-20
apple bee 0.22 1970-02-20
apple bee 0.22 1970-02-20
-PREHOOK: query: select * from newtypestbl where d in ('0.22', '11.22')
+PREHOOK: query: select * from newtypestbl where d in ('0.22', '11.22') sort by c
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-POSTHOOK: query: select * from newtypestbl where d in ('0.22', '11.22')
+POSTHOOK: query: select * from newtypestbl where d in ('0.22', '11.22') sort by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-PREHOOK: query: select * from newtypestbl where d in ('0.22', '11.22')
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+PREHOOK: query: select * from newtypestbl where d in ('0.22', '11.22') sort by c
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-POSTHOOK: query: select * from newtypestbl where d in ('0.22', '11.22')
+POSTHOOK: query: select * from newtypestbl where d in ('0.22', '11.22') sort by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
PREHOOK: query: select * from newtypestbl where d in ('0.9', '1.0')
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
@@ -590,42 +590,42 @@ apple bee 0.22 1970-02-20
apple bee 0.22 1970-02-20
apple bee 0.22 1970-02-20
apple bee 0.22 1970-02-20
-PREHOOK: query: select * from newtypestbl where d in ('0.9', 0.22, cast('11.22' as float))
+PREHOOK: query: select * from newtypestbl where d in ('0.9', 0.22, cast('11.22' as float)) sort by c
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-POSTHOOK: query: select * from newtypestbl where d in ('0.9', 0.22, cast('11.22' as float))
+POSTHOOK: query: select * from newtypestbl where d in ('0.9', 0.22, cast('11.22' as float)) sort by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-PREHOOK: query: select * from newtypestbl where d in ('0.9', 0.22, cast('11.22' as float))
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+PREHOOK: query: select * from newtypestbl where d in ('0.9', 0.22, cast('11.22' as float)) sort by c
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-POSTHOOK: query: select * from newtypestbl where d in ('0.9', 0.22, cast('11.22' as float))
+POSTHOOK: query: select * from newtypestbl where d in ('0.9', 0.22, cast('11.22' as float)) sort by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
PREHOOK: query: select * from newtypestbl where d between 0 and 1
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
@@ -652,42 +652,42 @@ apple bee 0.22 1970-02-20
apple bee 0.22 1970-02-20
apple bee 0.22 1970-02-20
apple bee 0.22 1970-02-20
-PREHOOK: query: select * from newtypestbl where d between 0 and 1000
+PREHOOK: query: select * from newtypestbl where d between 0 and 1000 sort by c
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-POSTHOOK: query: select * from newtypestbl where d between 0 and 1000
+POSTHOOK: query: select * from newtypestbl where d between 0 and 1000 sort by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-PREHOOK: query: select * from newtypestbl where d between 0 and 1000
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+PREHOOK: query: select * from newtypestbl where d between 0 and 1000 sort by c
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-POSTHOOK: query: select * from newtypestbl where d between 0 and 1000
+POSTHOOK: query: select * from newtypestbl where d between 0 and 1000 sort by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
PREHOOK: query: select * from newtypestbl where d between 0 and '2.0'
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
http://git-wip-us.apache.org/repos/asf/hive/blob/92b42ae9/ql/src/test/results/clientpositive/parquet_ppd_timestamp.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/parquet_ppd_timestamp.q.out b/ql/src/test/results/clientpositive/parquet_ppd_timestamp.q.out
index 3693879..e314c10 100644
--- a/ql/src/test/results/clientpositive/parquet_ppd_timestamp.q.out
+++ b/ql/src/test/results/clientpositive/parquet_ppd_timestamp.q.out
@@ -150,42 +150,42 @@ apple bee 0.22 2011-01-01 01:01:01
apple bee 0.22 2011-01-01 01:01:01
apple bee 0.22 2011-01-01 01:01:01
apple bee 0.22 2011-01-01 01:01:01
-PREHOOK: query: select * from newtypestbl where ts<cast('2011-01-22 01:01:01' as timestamp)
+PREHOOK: query: select * from newtypestbl where ts<cast('2011-01-22 01:01:01' as timestamp) sort by c
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-POSTHOOK: query: select * from newtypestbl where ts<cast('2011-01-22 01:01:01' as timestamp)
+POSTHOOK: query: select * from newtypestbl where ts<cast('2011-01-22 01:01:01' as timestamp) sort by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-hello world 11.22 2011-01-20 01:01:01
apple bee 0.22 2011-01-01 01:01:01
-hello world 11.22 2011-01-20 01:01:01
apple bee 0.22 2011-01-01 01:01:01
-hello world 11.22 2011-01-20 01:01:01
apple bee 0.22 2011-01-01 01:01:01
-hello world 11.22 2011-01-20 01:01:01
apple bee 0.22 2011-01-01 01:01:01
-hello world 11.22 2011-01-20 01:01:01
apple bee 0.22 2011-01-01 01:01:01
-PREHOOK: query: select * from newtypestbl where ts<cast('2011-01-22 01:01:01' as timestamp)
+hello world 11.22 2011-01-20 01:01:01
+hello world 11.22 2011-01-20 01:01:01
+hello world 11.22 2011-01-20 01:01:01
+hello world 11.22 2011-01-20 01:01:01
+hello world 11.22 2011-01-20 01:01:01
+PREHOOK: query: select * from newtypestbl where ts<cast('2011-01-22 01:01:01' as timestamp) sort by c
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-POSTHOOK: query: select * from newtypestbl where ts<cast('2011-01-22 01:01:01' as timestamp)
+POSTHOOK: query: select * from newtypestbl where ts<cast('2011-01-22 01:01:01' as timestamp) sort by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-hello world 11.22 2011-01-20 01:01:01
apple bee 0.22 2011-01-01 01:01:01
-hello world 11.22 2011-01-20 01:01:01
apple bee 0.22 2011-01-01 01:01:01
-hello world 11.22 2011-01-20 01:01:01
apple bee 0.22 2011-01-01 01:01:01
-hello world 11.22 2011-01-20 01:01:01
apple bee 0.22 2011-01-01 01:01:01
-hello world 11.22 2011-01-20 01:01:01
apple bee 0.22 2011-01-01 01:01:01
+hello world 11.22 2011-01-20 01:01:01
+hello world 11.22 2011-01-20 01:01:01
+hello world 11.22 2011-01-20 01:01:01
+hello world 11.22 2011-01-20 01:01:01
+hello world 11.22 2011-01-20 01:01:01
PREHOOK: query: select * from newtypestbl where ts<cast('2010-10-01 01:01:01' as timestamp)
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
@@ -228,42 +228,42 @@ apple bee 0.22 2011-01-01 01:01:01
apple bee 0.22 2011-01-01 01:01:01
apple bee 0.22 2011-01-01 01:01:01
apple bee 0.22 2011-01-01 01:01:01
-PREHOOK: query: select * from newtypestbl where ts<=cast('2011-01-20 01:01:01' as timestamp)
+PREHOOK: query: select * from newtypestbl where ts<=cast('2011-01-20 01:01:01' as timestamp) sort by c
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-POSTHOOK: query: select * from newtypestbl where ts<=cast('2011-01-20 01:01:01' as timestamp)
+POSTHOOK: query: select * from newtypestbl where ts<=cast('2011-01-20 01:01:01' as timestamp) sort by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-hello world 11.22 2011-01-20 01:01:01
apple bee 0.22 2011-01-01 01:01:01
-hello world 11.22 2011-01-20 01:01:01
apple bee 0.22 2011-01-01 01:01:01
-hello world 11.22 2011-01-20 01:01:01
apple bee 0.22 2011-01-01 01:01:01
-hello world 11.22 2011-01-20 01:01:01
apple bee 0.22 2011-01-01 01:01:01
-hello world 11.22 2011-01-20 01:01:01
apple bee 0.22 2011-01-01 01:01:01
-PREHOOK: query: select * from newtypestbl where ts<=cast('2011-01-20 01:01:01' as timestamp)
+hello world 11.22 2011-01-20 01:01:01
+hello world 11.22 2011-01-20 01:01:01
+hello world 11.22 2011-01-20 01:01:01
+hello world 11.22 2011-01-20 01:01:01
+hello world 11.22 2011-01-20 01:01:01
+PREHOOK: query: select * from newtypestbl where ts<=cast('2011-01-20 01:01:01' as timestamp) sort by c
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-POSTHOOK: query: select * from newtypestbl where ts<=cast('2011-01-20 01:01:01' as timestamp)
+POSTHOOK: query: select * from newtypestbl where ts<=cast('2011-01-20 01:01:01' as timestamp) sort by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-hello world 11.22 2011-01-20 01:01:01
apple bee 0.22 2011-01-01 01:01:01
-hello world 11.22 2011-01-20 01:01:01
apple bee 0.22 2011-01-01 01:01:01
-hello world 11.22 2011-01-20 01:01:01
apple bee 0.22 2011-01-01 01:01:01
-hello world 11.22 2011-01-20 01:01:01
apple bee 0.22 2011-01-01 01:01:01
-hello world 11.22 2011-01-20 01:01:01
apple bee 0.22 2011-01-01 01:01:01
+hello world 11.22 2011-01-20 01:01:01
+hello world 11.22 2011-01-20 01:01:01
+hello world 11.22 2011-01-20 01:01:01
+hello world 11.22 2011-01-20 01:01:01
+hello world 11.22 2011-01-20 01:01:01
PREHOOK: query: select * from newtypestbl where ts in (cast('2011-01-02 01:01:01' as timestamp), cast('2011-01-20 01:01:01' as timestamp))
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
@@ -290,42 +290,42 @@ hello world 11.22 2011-01-20 01:01:01
hello world 11.22 2011-01-20 01:01:01
hello world 11.22 2011-01-20 01:01:01
hello world 11.22 2011-01-20 01:01:01
-PREHOOK: query: select * from newtypestbl where ts in (cast('2011-01-01 01:01:01' as timestamp), cast('2011-01-20 01:01:01' as timestamp))
+PREHOOK: query: select * from newtypestbl where ts in (cast('2011-01-01 01:01:01' as timestamp), cast('2011-01-20 01:01:01' as timestamp)) sort by c
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-POSTHOOK: query: select * from newtypestbl where ts in (cast('2011-01-01 01:01:01' as timestamp), cast('2011-01-20 01:01:01' as timestamp))
+POSTHOOK: query: select * from newtypestbl where ts in (cast('2011-01-01 01:01:01' as timestamp), cast('2011-01-20 01:01:01' as timestamp)) sort by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-hello world 11.22 2011-01-20 01:01:01
apple bee 0.22 2011-01-01 01:01:01
-hello world 11.22 2011-01-20 01:01:01
apple bee 0.22 2011-01-01 01:01:01
-hello world 11.22 2011-01-20 01:01:01
apple bee 0.22 2011-01-01 01:01:01
-hello world 11.22 2011-01-20 01:01:01
apple bee 0.22 2011-01-01 01:01:01
-hello world 11.22 2011-01-20 01:01:01
apple bee 0.22 2011-01-01 01:01:01
-PREHOOK: query: select * from newtypestbl where ts in (cast('2011-01-01 01:01:01' as timestamp), cast('2011-01-20 01:01:01' as timestamp))
+hello world 11.22 2011-01-20 01:01:01
+hello world 11.22 2011-01-20 01:01:01
+hello world 11.22 2011-01-20 01:01:01
+hello world 11.22 2011-01-20 01:01:01
+hello world 11.22 2011-01-20 01:01:01
+PREHOOK: query: select * from newtypestbl where ts in (cast('2011-01-01 01:01:01' as timestamp), cast('2011-01-20 01:01:01' as timestamp)) sort by c
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-POSTHOOK: query: select * from newtypestbl where ts in (cast('2011-01-01 01:01:01' as timestamp), cast('2011-01-20 01:01:01' as timestamp))
+POSTHOOK: query: select * from newtypestbl where ts in (cast('2011-01-01 01:01:01' as timestamp), cast('2011-01-20 01:01:01' as timestamp)) sort by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-hello world 11.22 2011-01-20 01:01:01
apple bee 0.22 2011-01-01 01:01:01
-hello world 11.22 2011-01-20 01:01:01
apple bee 0.22 2011-01-01 01:01:01
-hello world 11.22 2011-01-20 01:01:01
apple bee 0.22 2011-01-01 01:01:01
-hello world 11.22 2011-01-20 01:01:01
apple bee 0.22 2011-01-01 01:01:01
-hello world 11.22 2011-01-20 01:01:01
apple bee 0.22 2011-01-01 01:01:01
+hello world 11.22 2011-01-20 01:01:01
+hello world 11.22 2011-01-20 01:01:01
+hello world 11.22 2011-01-20 01:01:01
+hello world 11.22 2011-01-20 01:01:01
+hello world 11.22 2011-01-20 01:01:01
PREHOOK: query: select * from newtypestbl where ts in (cast('2011-01-02 01:01:01' as timestamp), cast('2011-01-08 01:01:01' as timestamp))
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
@@ -368,42 +368,42 @@ apple bee 0.22 2011-01-01 01:01:01
apple bee 0.22 2011-01-01 01:01:01
apple bee 0.22 2011-01-01 01:01:01
apple bee 0.22 2011-01-01 01:01:01
-PREHOOK: query: select * from newtypestbl where ts between cast('2010-10-01 01:01:01' as timestamp) and cast('2011-01-25 01:01:01' as timestamp)
+PREHOOK: query: select * from newtypestbl where ts between cast('2010-10-01 01:01:01' as timestamp) and cast('2011-01-25 01:01:01' as timestamp) sort by c
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-POSTHOOK: query: select * from newtypestbl where ts between cast('2010-10-01 01:01:01' as timestamp) and cast('2011-01-25 01:01:01' as timestamp)
+POSTHOOK: query: select * from newtypestbl where ts between cast('2010-10-01 01:01:01' as timestamp) and cast('2011-01-25 01:01:01' as timestamp) sort by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-hello world 11.22 2011-01-20 01:01:01
apple bee 0.22 2011-01-01 01:01:01
-hello world 11.22 2011-01-20 01:01:01
apple bee 0.22 2011-01-01 01:01:01
-hello world 11.22 2011-01-20 01:01:01
apple bee 0.22 2011-01-01 01:01:01
-hello world 11.22 2011-01-20 01:01:01
apple bee 0.22 2011-01-01 01:01:01
-hello world 11.22 2011-01-20 01:01:01
apple bee 0.22 2011-01-01 01:01:01
-PREHOOK: query: select * from newtypestbl where ts between cast('2010-10-01 01:01:01' as timestamp) and cast('2011-01-25 01:01:01' as timestamp)
+hello world 11.22 2011-01-20 01:01:01
+hello world 11.22 2011-01-20 01:01:01
+hello world 11.22 2011-01-20 01:01:01
+hello world 11.22 2011-01-20 01:01:01
+hello world 11.22 2011-01-20 01:01:01
+PREHOOK: query: select * from newtypestbl where ts between cast('2010-10-01 01:01:01' as timestamp) and cast('2011-01-25 01:01:01' as timestamp) sort by c
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-POSTHOOK: query: select * from newtypestbl where ts between cast('2010-10-01 01:01:01' as timestamp) and cast('2011-01-25 01:01:01' as timestamp)
+POSTHOOK: query: select * from newtypestbl where ts between cast('2010-10-01 01:01:01' as timestamp) and cast('2011-01-25 01:01:01' as timestamp) sort by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-hello world 11.22 2011-01-20 01:01:01
apple bee 0.22 2011-01-01 01:01:01
-hello world 11.22 2011-01-20 01:01:01
apple bee 0.22 2011-01-01 01:01:01
-hello world 11.22 2011-01-20 01:01:01
apple bee 0.22 2011-01-01 01:01:01
-hello world 11.22 2011-01-20 01:01:01
apple bee 0.22 2011-01-01 01:01:01
-hello world 11.22 2011-01-20 01:01:01
apple bee 0.22 2011-01-01 01:01:01
+hello world 11.22 2011-01-20 01:01:01
+hello world 11.22 2011-01-20 01:01:01
+hello world 11.22 2011-01-20 01:01:01
+hello world 11.22 2011-01-20 01:01:01
+hello world 11.22 2011-01-20 01:01:01
PREHOOK: query: select * from newtypestbl where ts between cast('2010-10-01 01:01:01' as timestamp) and cast('2010-11-01 01:01:01' as timestamp)
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
http://git-wip-us.apache.org/repos/asf/hive/blob/92b42ae9/ql/src/test/results/clientpositive/parquet_ppd_varchar.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/parquet_ppd_varchar.q.out b/ql/src/test/results/clientpositive/parquet_ppd_varchar.q.out
index 0574e5d..2e9f72f 100644
--- a/ql/src/test/results/clientpositive/parquet_ppd_varchar.q.out
+++ b/ql/src/test/results/clientpositive/parquet_ppd_varchar.q.out
@@ -98,42 +98,42 @@ apple bee 0.22 1970-02-20
apple bee 0.22 1970-02-20
apple bee 0.22 1970-02-20
apple bee 0.22 1970-02-20
-PREHOOK: query: select * from newtypestbl where v<="world"
+PREHOOK: query: select * from newtypestbl where v<="world" sort by c
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-POSTHOOK: query: select * from newtypestbl where v<="world"
+POSTHOOK: query: select * from newtypestbl where v<="world" sort by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-PREHOOK: query: select * from newtypestbl where v<="world"
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+PREHOOK: query: select * from newtypestbl where v<="world" sort by c
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-POSTHOOK: query: select * from newtypestbl where v<="world"
+POSTHOOK: query: select * from newtypestbl where v<="world" sort by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
PREHOOK: query: select * from newtypestbl where v="bee "
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
@@ -176,42 +176,42 @@ apple bee 0.22 1970-02-20
apple bee 0.22 1970-02-20
apple bee 0.22 1970-02-20
apple bee 0.22 1970-02-20
-PREHOOK: query: select * from newtypestbl where v in ("bee", "world")
+PREHOOK: query: select * from newtypestbl where v in ("bee", "world") sort by c
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-POSTHOOK: query: select * from newtypestbl where v in ("bee", "world")
+POSTHOOK: query: select * from newtypestbl where v in ("bee", "world") sort by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-PREHOOK: query: select * from newtypestbl where v in ("bee", "world")
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+PREHOOK: query: select * from newtypestbl where v in ("bee", "world") sort by c
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-POSTHOOK: query: select * from newtypestbl where v in ("bee", "world")
+POSTHOOK: query: select * from newtypestbl where v in ("bee", "world") sort by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
PREHOOK: query: select * from newtypestbl where v in ("orange")
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
@@ -254,42 +254,42 @@ apple bee 0.22 1970-02-20
apple bee 0.22 1970-02-20
apple bee 0.22 1970-02-20
apple bee 0.22 1970-02-20
-PREHOOK: query: select * from newtypestbl where v between "bee" and "zombie"
+PREHOOK: query: select * from newtypestbl where v between "bee" and "zombie" sort by c
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-POSTHOOK: query: select * from newtypestbl where v between "bee" and "zombie"
+POSTHOOK: query: select * from newtypestbl where v between "bee" and "zombie" sort by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-PREHOOK: query: select * from newtypestbl where v between "bee" and "zombie"
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+PREHOOK: query: select * from newtypestbl where v between "bee" and "zombie" sort by c
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-POSTHOOK: query: select * from newtypestbl where v between "bee" and "zombie"
+POSTHOOK: query: select * from newtypestbl where v between "bee" and "zombie" sort by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypestbl
#### A masked pattern was here ####
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
-hello world 11.22 1970-02-27
apple bee 0.22 1970-02-20
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
+hello world 11.22 1970-02-27
PREHOOK: query: select * from newtypestbl where v between "orange" and "pine"
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypestbl
[2/8] hive git commit: HIVE-11820 : export tables with size of >32MB
throws java.lang.IllegalArgumentException: Skip CRC is valid only with update
options (Takahiko Saito via Ashutosh Chauhan)
Posted by se...@apache.org.
HIVE-11820 : export tables with size of >32MB throws java.lang.IllegalArgumentException: Skip CRC is valid only with update options (Takahiko Saito via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/93a66276
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/93a66276
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/93a66276
Branch: refs/heads/llap
Commit: 93a6627606cb2dad0e04c3a885f71c1be405f51d
Parents: cc78dd5
Author: Takahiko Saito <ty...@gmail.com>
Authored: Wed Sep 16 15:34:00 2015 -0800
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Sat Sep 19 23:50:49 2015 -0700
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java | 5 ++++-
.../main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java | 4 +++-
2 files changed, 7 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/93a66276/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
----------------------------------------------------------------------
diff --git a/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java b/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
index 6c6ccbc..93dcbd3 100644
--- a/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
+++ b/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
@@ -73,6 +73,8 @@ import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.tools.distcp2.DistCp;
import org.apache.hadoop.tools.distcp2.DistCpOptions;
+import org.apache.hadoop.tools.distcp2.DistCpOptions.FileAttribute;
+
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.VersionInfo;
@@ -672,8 +674,9 @@ public class Hadoop20SShims extends HadoopShimsSecure {
public boolean runDistCp(Path src, Path dst, Configuration conf) throws IOException {
DistCpOptions options = new DistCpOptions(Collections.singletonList(src), dst);
- options.setSkipCRC(true);
options.setSyncFolder(true);
+ options.setSkipCRC(true);
+ options.preserve(FileAttribute.BLOCKSIZE);
try {
DistCp distcp = new DistCp(conf, options);
distcp.execute();
http://git-wip-us.apache.org/repos/asf/hive/blob/93a66276/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
----------------------------------------------------------------------
diff --git a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
index 3292cb3..83369ee 100644
--- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
+++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
@@ -97,6 +97,7 @@ import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.tools.DistCp;
import org.apache.hadoop.tools.DistCpOptions;
+import org.apache.hadoop.tools.DistCpOptions.FileAttribute;
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.tez.test.MiniTezCluster;
@@ -1213,8 +1214,9 @@ public class Hadoop23Shims extends HadoopShimsSecure {
public boolean runDistCp(Path src, Path dst, Configuration conf) throws IOException {
DistCpOptions options = new DistCpOptions(Collections.singletonList(src), dst);
- options.setSkipCRC(true);
options.setSyncFolder(true);
+ options.setSkipCRC(true);
+ options.preserve(FileAttribute.BLOCKSIZE);
try {
DistCp distcp = new DistCp(conf, options);
distcp.execute();
[3/8] hive git commit: HIVE-11891 - Add basic performance logging to
metastore calls (Brock via Szehon)
Posted by se...@apache.org.
HIVE-11891 - Add basic performance logging to metastore calls (Brock via Szehon)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/21861592
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/21861592
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/21861592
Branch: refs/heads/llap
Commit: 2186159209db49f2aeab06be7c38203fbbb5550c
Parents: 93a6627
Author: Brock Noland <br...@apache.org>
Authored: Sun Sep 20 15:49:01 2015 -0700
Committer: Brock Noland <br...@apache.org>
Committed: Sun Sep 20 15:49:01 2015 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/log/PerfLogger.java | 196 +++++++++++++++++++
.../hive/metastore/RetryingHMSHandler.java | 33 +++-
.../java/org/apache/hadoop/hive/ql/Driver.java | 11 +-
.../hadoop/hive/ql/exec/MapJoinOperator.java | 3 +-
.../ql/exec/SparkHashTableSinkOperator.java | 3 +-
.../apache/hadoop/hive/ql/exec/Utilities.java | 11 +-
.../hadoop/hive/ql/exec/spark/SparkPlan.java | 3 +-
.../hive/ql/exec/spark/SparkPlanGenerator.java | 3 +-
.../hive/ql/exec/spark/SparkRecordHandler.java | 3 +-
.../hadoop/hive/ql/exec/spark/SparkTask.java | 2 +-
.../ql/exec/spark/status/SparkJobMonitor.java | 2 +-
.../hive/ql/exec/tez/RecordProcessor.java | 3 +-
.../hive/ql/exec/tez/ReduceRecordProcessor.java | 1 -
.../hive/ql/exec/tez/ReduceRecordSource.java | 3 +-
.../hadoop/hive/ql/exec/tez/TezJobMonitor.java | 4 +-
.../hadoop/hive/ql/exec/tez/TezProcessor.java | 3 +-
.../apache/hadoop/hive/ql/exec/tez/TezTask.java | 2 +-
.../hive/ql/io/CombineHiveInputFormat.java | 10 +-
.../hadoop/hive/ql/io/HiveInputFormat.java | 5 +-
.../apache/hadoop/hive/ql/log/PerfLogger.java | 195 ------------------
.../hive/ql/optimizer/ppr/PartitionPruner.java | 7 +-
.../hive/ql/parse/spark/SparkCompiler.java | 3 +-
.../hadoop/hive/ql/session/SessionState.java | 37 ++--
23 files changed, 293 insertions(+), 250 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/21861592/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java b/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java
new file mode 100644
index 0000000..6263a6d
--- /dev/null
+++ b/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java
@@ -0,0 +1,196 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.log;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.util.ReflectionUtils;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * PerfLogger.
+ *
+ * Can be used to measure and log the time spent by a piece of code.
+ */
+public class PerfLogger {
+ public static final String ACQUIRE_READ_WRITE_LOCKS = "acquireReadWriteLocks";
+ public static final String COMPILE = "compile";
+ public static final String PARSE = "parse";
+ public static final String ANALYZE = "semanticAnalyze";
+ public static final String DO_AUTHORIZATION = "doAuthorization";
+ public static final String DRIVER_EXECUTE = "Driver.execute";
+ public static final String INPUT_SUMMARY = "getInputSummary";
+ public static final String GET_SPLITS = "getSplits";
+ public static final String RUN_TASKS = "runTasks";
+ public static final String SERIALIZE_PLAN = "serializePlan";
+ public static final String DESERIALIZE_PLAN = "deserializePlan";
+ public static final String CLONE_PLAN = "clonePlan";
+ public static final String TASK = "task.";
+ public static final String RELEASE_LOCKS = "releaseLocks";
+ public static final String PRUNE_LISTING = "prune-listing";
+ public static final String PARTITION_RETRIEVING = "partition-retrieving";
+ public static final String PRE_HOOK = "PreHook.";
+ public static final String POST_HOOK = "PostHook.";
+ public static final String FAILURE_HOOK = "FailureHook.";
+ public static final String DRIVER_RUN = "Driver.run";
+ public static final String TIME_TO_SUBMIT = "TimeToSubmit";
+ public static final String TEZ_SUBMIT_TO_RUNNING = "TezSubmitToRunningDag";
+ public static final String TEZ_BUILD_DAG = "TezBuildDag";
+ public static final String TEZ_SUBMIT_DAG = "TezSubmitDag";
+ public static final String TEZ_RUN_DAG = "TezRunDag";
+ public static final String TEZ_CREATE_VERTEX = "TezCreateVertex.";
+ public static final String TEZ_RUN_VERTEX = "TezRunVertex.";
+ public static final String TEZ_INITIALIZE_PROCESSOR = "TezInitializeProcessor";
+ public static final String TEZ_RUN_PROCESSOR = "TezRunProcessor";
+ public static final String TEZ_INIT_OPERATORS = "TezInitializeOperators";
+ public static final String LOAD_HASHTABLE = "LoadHashtable";
+
+ public static final String SPARK_SUBMIT_TO_RUNNING = "SparkSubmitToRunning";
+ public static final String SPARK_BUILD_PLAN = "SparkBuildPlan";
+ public static final String SPARK_BUILD_RDD_GRAPH = "SparkBuildRDDGraph";
+ public static final String SPARK_SUBMIT_JOB = "SparkSubmitJob";
+ public static final String SPARK_RUN_JOB = "SparkRunJob";
+ public static final String SPARK_CREATE_TRAN = "SparkCreateTran.";
+ public static final String SPARK_RUN_STAGE = "SparkRunStage.";
+ public static final String SPARK_INIT_OPERATORS = "SparkInitializeOperators";
+ public static final String SPARK_GENERATE_TASK_TREE = "SparkGenerateTaskTree";
+ public static final String SPARK_OPTIMIZE_OPERATOR_TREE = "SparkOptimizeOperatorTree";
+ public static final String SPARK_OPTIMIZE_TASK_TREE = "SparkOptimizeTaskTree";
+ public static final String SPARK_FLUSH_HASHTABLE = "SparkFlushHashTable.";
+
+ protected final Map<String, Long> startTimes = new HashMap<String, Long>();
+ protected final Map<String, Long> endTimes = new HashMap<String, Long>();
+
+ static final private Log LOG = LogFactory.getLog(PerfLogger.class.getName());
+ protected static final ThreadLocal<PerfLogger> perfLogger = new ThreadLocal<PerfLogger>();
+
+
+ public PerfLogger() {
+ // Use getPerfLogger to get an instance of PerfLogger
+ }
+
+ public static PerfLogger getPerfLogger(HiveConf conf, boolean resetPerfLogger) {
+ PerfLogger result = perfLogger.get();
+ if (resetPerfLogger || result == null) {
+ if (conf == null) {
+ result = new PerfLogger();
+ } else {
+ try {
+ result = (PerfLogger) ReflectionUtils.newInstance(conf.getClassByName(
+ conf.getVar(HiveConf.ConfVars.HIVE_PERF_LOGGER)), conf);
+ } catch (ClassNotFoundException e) {
+ LOG.error("Performance Logger Class not found:" + e.getMessage());
+ result = new PerfLogger();
+ }
+ }
+ perfLogger.set(result);
+ }
+ return result;
+ }
+
+ /**
+ * Call this function when you start to measure time spent by a piece of code.
+ * @param callerName the logging object to be used.
+ * @param method method or ID that identifies this perf log element.
+ */
+ public void PerfLogBegin(String callerName, String method) {
+ long startTime = System.currentTimeMillis();
+ LOG.info("<PERFLOG method=" + method + " from=" + callerName + ">");
+ startTimes.put(method, new Long(startTime));
+ }
+ /**
+ * Call this function in correspondence of PerfLogBegin to mark the end of the measurement.
+ * @param callerName
+ * @param method
+ * @return long duration the difference between now and startTime, or -1 if startTime is null
+ */
+ public long PerfLogEnd(String callerName, String method) {
+ return PerfLogEnd(callerName, method, null);
+ }
+
+ /**
+ * Call this function in correspondence of PerfLogBegin to mark the end of the measurement.
+ * @param callerName
+ * @param method
+ * @return long duration the difference between now and startTime, or -1 if startTime is null
+ */
+ public long PerfLogEnd(String callerName, String method, String additionalInfo) {
+ Long startTime = startTimes.get(method);
+ long endTime = System.currentTimeMillis();
+ long duration = -1;
+
+ endTimes.put(method, new Long(endTime));
+
+ StringBuilder sb = new StringBuilder("</PERFLOG method=").append(method);
+ if (startTime != null) {
+ sb.append(" start=").append(startTime);
+ }
+ sb.append(" end=").append(endTime);
+ if (startTime != null) {
+ duration = endTime - startTime.longValue();
+ sb.append(" duration=").append(duration);
+ }
+ sb.append(" from=").append(callerName);
+ if (additionalInfo != null) {
+ sb.append(" ").append(additionalInfo);
+ }
+ sb.append(">");
+ LOG.info(sb);
+
+ return duration;
+ }
+
+ public Long getStartTime(String method) {
+ long startTime = 0L;
+
+ if (startTimes.containsKey(method)) {
+ startTime = startTimes.get(method);
+ }
+ return startTime;
+ }
+
+ public Long getEndTime(String method) {
+ long endTime = 0L;
+
+ if (endTimes.containsKey(method)) {
+ endTime = endTimes.get(method);
+ }
+ return endTime;
+ }
+
+ public boolean startTimeHasMethod(String method) {
+ return startTimes.containsKey(method);
+ }
+
+ public boolean endTimeHasMethod(String method) {
+ return endTimes.containsKey(method);
+ }
+
+ public Long getDuration(String method) {
+ long duration = 0;
+ if (startTimes.containsKey(method) && endTimes.containsKey(method)) {
+ duration = endTimes.get(method) - startTimes.get(method);
+ }
+ return duration;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/21861592/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java
index 892aef4..56276b6 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.hive.common.classification.InterfaceStability;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
+import org.apache.hadoop.hive.ql.log.PerfLogger;
import org.datanucleus.exceptions.NucleusException;
@InterfaceAudience.Private
@@ -41,6 +42,17 @@ import org.datanucleus.exceptions.NucleusException;
public class RetryingHMSHandler implements InvocationHandler {
private static final Log LOG = LogFactory.getLog(RetryingHMSHandler.class);
+ private static final String CLASS_NAME = RetryingHMSHandler.class.getName();
+
+ private static class Result {
+ private final Object result;
+ private final int numRetries;
+
+ public Result(Object result, int numRetries) {
+ this.result = result;
+ this.numRetries = numRetries;
+ }
+ }
private final IHMSHandler baseHandler;
private final MetaStoreInit.MetaStoreInitData metaStoreInitData =
@@ -78,6 +90,25 @@ public class RetryingHMSHandler implements InvocationHandler {
@Override
public Object invoke(final Object proxy, final Method method, final Object[] args) throws Throwable {
+ int retryCount = -1;
+ int threadId = HiveMetaStore.HMSHandler.get();
+ boolean error = true;
+ PerfLogger perfLogger = PerfLogger.getPerfLogger(origConf, false);
+ perfLogger.PerfLogBegin(CLASS_NAME, method.getName());
+ try {
+ Result result = invokeInternal(proxy, method, args);
+ retryCount = result.numRetries;
+ error = false;
+ return result.result;
+ } finally {
+ StringBuffer additionalInfo = new StringBuffer();
+ additionalInfo.append("threadId=").append(threadId).append(" retryCount=").append(retryCount)
+ .append(" error=").append(error);
+ perfLogger.PerfLogEnd(CLASS_NAME, method.getName(), additionalInfo.toString());
+ }
+ }
+
+ public Result invokeInternal(final Object proxy, final Method method, final Object[] args) throws Throwable {
boolean gotNewConnectUrl = false;
boolean reloadConf = HiveConf.getBoolVar(origConf,
@@ -106,7 +137,7 @@ public class RetryingHMSHandler implements InvocationHandler {
Deadline.startTimer(method.getName());
Object object = method.invoke(baseHandler, args);
Deadline.stopTimer();
- return object;
+ return new Result(object, retryCount);
} catch (javax.jdo.JDOException e) {
caughtException = e;
http://git-wip-us.apache.org/repos/asf/hive/blob/21861592/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index 4030075..43159c6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -364,7 +364,7 @@ public class Driver implements CommandProcessor {
* @return 0 for ok
*/
public int compile(String command, boolean resetTaskIds) {
- PerfLogger perfLogger = PerfLogger.getPerfLogger();
+ PerfLogger perfLogger = SessionState.getPerfLogger();
perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.COMPILE);
//holder for parent command type/string when executing reentrant queries
@@ -953,7 +953,7 @@ public class Driver implements CommandProcessor {
* @param startTxnImplicitly in AC=false, the 1st DML starts a txn
**/
private int acquireLocksAndOpenTxn(boolean startTxnImplicitly) {
- PerfLogger perfLogger = PerfLogger.getPerfLogger();
+ PerfLogger perfLogger = SessionState.getPerfLogger();
perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.ACQUIRE_READ_WRITE_LOCKS);
SessionState ss = SessionState.get();
@@ -1039,7 +1039,7 @@ public class Driver implements CommandProcessor {
**/
private void releaseLocksAndCommitOrRollback(List<HiveLock> hiveLocks, boolean commit)
throws LockException {
- PerfLogger perfLogger = PerfLogger.getPerfLogger();
+ PerfLogger perfLogger = SessionState.getPerfLogger();
perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.RELEASE_LOCKS);
SessionState ss = SessionState.get();
@@ -1194,7 +1194,7 @@ public class Driver implements CommandProcessor {
}
// Reset the perf logger
- PerfLogger perfLogger = PerfLogger.getPerfLogger(true);
+ PerfLogger perfLogger = SessionState.getPerfLogger(true);
perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.DRIVER_RUN);
perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.TIME_TO_SUBMIT);
@@ -1282,7 +1282,6 @@ public class Driver implements CommandProcessor {
}
perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.DRIVER_RUN);
- perfLogger.close(LOG, plan);
// Take all the driver run hooks and post-execute them.
try {
@@ -1406,7 +1405,7 @@ public class Driver implements CommandProcessor {
}
public int execute() throws CommandNeedRetryException {
- PerfLogger perfLogger = PerfLogger.getPerfLogger();
+ PerfLogger perfLogger = SessionState.getPerfLogger();
perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.DRIVER_EXECUTE);
boolean noName = StringUtils.isEmpty(conf.getVar(HiveConf.ConfVars.HADOOPJOBNAME));
int maxlen = conf.getIntVar(HiveConf.ConfVars.HIVEJOBNAMELENGTH);
http://git-wip-us.apache.org/repos/asf/hive/blob/21861592/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
index a9159a5..02d61eb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
@@ -60,6 +60,7 @@ import org.apache.hadoop.hive.ql.plan.JoinDesc;
import org.apache.hadoop.hive.ql.plan.MapJoinDesc;
import org.apache.hadoop.hive.ql.plan.TableDesc;
import org.apache.hadoop.hive.ql.plan.api.OperatorType;
+import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.serde2.SerDe;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.SerDeUtils;
@@ -76,7 +77,7 @@ public class MapJoinOperator extends AbstractMapJoinOperator<MapJoinDesc> implem
private static final long serialVersionUID = 1L;
private static final Log LOG = LogFactory.getLog(MapJoinOperator.class.getName());
private static final String CLASS_NAME = MapJoinOperator.class.getName();
- private final PerfLogger perfLogger = PerfLogger.getPerfLogger();
+ private final PerfLogger perfLogger = SessionState.getPerfLogger();
private transient String cacheKey;
private transient ObjectCache cache;
http://git-wip-us.apache.org/repos/asf/hive/blob/21861592/ql/src/java/org/apache/hadoop/hive/ql/exec/SparkHashTableSinkOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/SparkHashTableSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/SparkHashTableSinkOperator.java
index aa8808a..af368eb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/SparkHashTableSinkOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/SparkHashTableSinkOperator.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.hive.ql.plan.MapredLocalWork;
import org.apache.hadoop.hive.ql.plan.SparkBucketMapJoinContext;
import org.apache.hadoop.hive.ql.plan.SparkHashTableSinkDesc;
import org.apache.hadoop.hive.ql.plan.api.OperatorType;
+import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
public class SparkHashTableSinkOperator
@@ -48,7 +49,7 @@ public class SparkHashTableSinkOperator
private static final int MIN_REPLICATION = 10;
private static final long serialVersionUID = 1L;
private final String CLASS_NAME = this.getClass().getName();
- private final PerfLogger perfLogger = PerfLogger.getPerfLogger();
+ private final PerfLogger perfLogger = SessionState.getPerfLogger();
protected static final Log LOG = LogFactory.getLog(SparkHashTableSinkOperator.class.getName());
private final HashTableSinkOperator htsOperator;
http://git-wip-us.apache.org/repos/asf/hive/blob/21861592/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index ca86301..bcf85a4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -82,7 +82,6 @@ import java.util.zip.DeflaterOutputStream;
import java.util.zip.InflaterInputStream;
import org.antlr.runtime.CommonToken;
-import org.apache.calcite.util.ChunkList;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.WordUtils;
@@ -936,7 +935,7 @@ public final class Utilities {
}
private static void serializePlan(Object plan, OutputStream out, Configuration conf, boolean cloningPlan) {
- PerfLogger perfLogger = PerfLogger.getPerfLogger();
+ PerfLogger perfLogger = SessionState.getPerfLogger();
perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.SERIALIZE_PLAN);
String serializationType = conf.get(HiveConf.ConfVars.PLAN_SERIALIZATION.varname, "kryo");
LOG.info("Serializing " + plan.getClass().getSimpleName() + " via " + serializationType);
@@ -962,7 +961,7 @@ public final class Utilities {
}
private static <T> T deserializePlan(InputStream in, Class<T> planClass, Configuration conf, boolean cloningPlan) {
- PerfLogger perfLogger = PerfLogger.getPerfLogger();
+ PerfLogger perfLogger = SessionState.getPerfLogger();
perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.DESERIALIZE_PLAN);
T plan;
String serializationType = conf.get(HiveConf.ConfVars.PLAN_SERIALIZATION.varname, "kryo");
@@ -997,7 +996,7 @@ public final class Utilities {
*/
public static MapredWork clonePlan(MapredWork plan) {
// TODO: need proper clone. Meanwhile, let's at least keep this horror in one place
- PerfLogger perfLogger = PerfLogger.getPerfLogger();
+ PerfLogger perfLogger = SessionState.getPerfLogger();
perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.CLONE_PLAN);
ByteArrayOutputStream baos = new ByteArrayOutputStream(4096);
Configuration conf = new HiveConf();
@@ -1014,7 +1013,7 @@ public final class Utilities {
* @return The clone.
*/
public static BaseWork cloneBaseWork(BaseWork plan) {
- PerfLogger perfLogger = PerfLogger.getPerfLogger();
+ PerfLogger perfLogger = SessionState.getPerfLogger();
perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.CLONE_PLAN);
ByteArrayOutputStream baos = new ByteArrayOutputStream(4096);
Configuration conf = new HiveConf();
@@ -2530,7 +2529,7 @@ public final class Utilities {
*/
public static ContentSummary getInputSummary(final Context ctx, MapWork work, PathFilter filter)
throws IOException {
- PerfLogger perfLogger = PerfLogger.getPerfLogger();
+ PerfLogger perfLogger = SessionState.getPerfLogger();
perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.INPUT_SUMMARY);
long[] summary = {0, 0, 0};
http://git-wip-us.apache.org/repos/asf/hive/blob/21861592/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlan.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlan.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlan.java
index daf9698..9906118 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlan.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlan.java
@@ -30,6 +30,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.ql.io.HiveKey;
import org.apache.hadoop.hive.ql.log.PerfLogger;
+import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.io.BytesWritable;
import org.apache.spark.api.java.JavaPairRDD;
@@ -39,7 +40,7 @@ import com.google.common.base.Preconditions;
public class SparkPlan {
private static final String CLASS_NAME = SparkPlan.class.getName();
private static final Log LOG = LogFactory.getLog(SparkPlan.class);
- private final PerfLogger perfLogger = PerfLogger.getPerfLogger();
+ private final PerfLogger perfLogger = SessionState.getPerfLogger();
private final Set<SparkTran> rootTrans = new HashSet<SparkTran>();
private final Set<SparkTran> leafTrans = new HashSet<SparkTran>();
http://git-wip-us.apache.org/repos/asf/hive/blob/21861592/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java
index 762ce7d..4c3ee4b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.hive.ql.io.merge.MergeFileMapper;
import org.apache.hadoop.hive.ql.io.merge.MergeFileOutputFormat;
import org.apache.hadoop.hive.ql.io.merge.MergeFileWork;
import org.apache.hadoop.hive.ql.log.PerfLogger;
+import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.Partitioner;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -61,7 +62,7 @@ import org.apache.spark.api.java.JavaSparkContext;
@SuppressWarnings("rawtypes")
public class SparkPlanGenerator {
private static final String CLASS_NAME = SparkPlanGenerator.class.getName();
- private final PerfLogger perfLogger = PerfLogger.getPerfLogger();
+ private final PerfLogger perfLogger = SessionState.getPerfLogger();
private static final Log LOG = LogFactory.getLog(SparkPlanGenerator.class);
private JavaSparkContext sc;
http://git-wip-us.apache.org/repos/asf/hive/blob/21861592/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkRecordHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkRecordHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkRecordHandler.java
index 97b3471..3d37753 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkRecordHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkRecordHandler.java
@@ -22,6 +22,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.ql.exec.MapredContext;
import org.apache.hadoop.hive.ql.log.PerfLogger;
+import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;
@@ -35,7 +36,7 @@ import java.util.Iterator;
public abstract class SparkRecordHandler {
protected static final String CLASS_NAME = SparkRecordHandler.class.getName();
- protected final PerfLogger perfLogger = PerfLogger.getPerfLogger();
+ protected final PerfLogger perfLogger = SessionState.getPerfLogger();
private static final Log LOG = LogFactory.getLog(SparkRecordHandler.class);
// used to log memory usage periodically
http://git-wip-us.apache.org/repos/asf/hive/blob/21861592/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java
index a36dc6e..eac812f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java
@@ -80,7 +80,7 @@ public class SparkTask extends Task<SparkWork> {
private static final String CLASS_NAME = SparkTask.class.getName();
private static final Log LOG = LogFactory.getLog(CLASS_NAME);
private static final LogHelper console = new LogHelper(LOG);
- private final PerfLogger perfLogger = PerfLogger.getPerfLogger();
+ private final PerfLogger perfLogger = SessionState.getPerfLogger();
private static final long serialVersionUID = 1L;
private SparkCounters sparkCounters;
http://git-wip-us.apache.org/repos/asf/hive/blob/21861592/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java
index 3fceeb0..6fc20c7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java
@@ -38,7 +38,7 @@ abstract class SparkJobMonitor {
protected static final String CLASS_NAME = SparkJobMonitor.class.getName();
protected static final Log LOG = LogFactory.getLog(CLASS_NAME);
protected static SessionState.LogHelper console = new SessionState.LogHelper(LOG);
- protected final PerfLogger perfLogger = PerfLogger.getPerfLogger();
+ protected final PerfLogger perfLogger = SessionState.getPerfLogger();
protected final int checkInterval = 1000;
protected final long monitorTimeoutInteval;
http://git-wip-us.apache.org/repos/asf/hive/blob/21861592/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/RecordProcessor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/RecordProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/RecordProcessor.java
index c563d9d..87fded1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/RecordProcessor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/RecordProcessor.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.hive.ql.log.PerfLogger;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.BaseWork;
import org.apache.hadoop.hive.ql.plan.MapWork;
+import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.tez.mapreduce.processor.MRTaskReporter;
@@ -64,7 +65,7 @@ public abstract class RecordProcessor {
protected boolean isLogTraceEnabled = false;
protected MRTaskReporter reporter;
- protected PerfLogger perfLogger = PerfLogger.getPerfLogger();
+ protected PerfLogger perfLogger = SessionState.getPerfLogger();
protected String CLASS_NAME = RecordProcessor.class.getName();
public RecordProcessor(JobConf jConf, ProcessorContext processorContext) {
http://git-wip-us.apache.org/repos/asf/hive/blob/21861592/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java
index d649672..91ba2bb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java
@@ -51,7 +51,6 @@ import org.apache.tez.runtime.api.LogicalInput;
import org.apache.tez.runtime.api.LogicalOutput;
import org.apache.tez.runtime.api.ProcessorContext;
import org.apache.tez.runtime.api.Reader;
-import org.apache.tez.runtime.library.api.KeyValuesReader;
/**
* Process input from tez LogicalInput and write output - for a map plan
http://git-wip-us.apache.org/repos/asf/hive/blob/21861592/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java
index 89f7572..1f2f9f9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriterF
import org.apache.hadoop.hive.ql.log.PerfLogger;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.TableDesc;
+import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.serde2.Deserializer;
import org.apache.hadoop.hive.serde2.SerDe;
import org.apache.hadoop.hive.serde2.SerDeException;
@@ -115,7 +116,7 @@ public class ReduceRecordSource implements RecordSource {
private ObjectInspector valueObjectInspector;
- private final PerfLogger perfLogger = PerfLogger.getPerfLogger();
+ private final PerfLogger perfLogger = SessionState.getPerfLogger();
private Iterable<Object> valueWritables;
http://git-wip-us.apache.org/repos/asf/hive/blob/21861592/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
index 1e1603b..754c332 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
@@ -61,8 +61,6 @@ import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
-import jline.TerminalFactory;
-
/**
* TezJobMonitor keeps track of a tez job while it's being executed. It will
* print status to the console and retrieve final status of the job after
@@ -100,7 +98,7 @@ public class TezJobMonitor {
private String separator;
private transient LogHelper console;
- private final PerfLogger perfLogger = PerfLogger.getPerfLogger();
+ private final PerfLogger perfLogger = SessionState.getPerfLogger();
private final int checkInterval = 200;
private final int maxRetryInterval = 2500;
private final int printInterval = 3000;
http://git-wip-us.apache.org/repos/asf/hive/blob/21861592/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezProcessor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezProcessor.java
index 39f9db6..fad79f8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezProcessor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezProcessor.java
@@ -26,6 +26,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.ql.log.PerfLogger;
+import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.util.StringUtils;
@@ -54,7 +55,7 @@ public class TezProcessor extends AbstractLogicalIOProcessor {
protected JobConf jobConf;
private static final String CLASS_NAME = TezProcessor.class.getName();
- private final PerfLogger perfLogger = PerfLogger.getPerfLogger();
+ private final PerfLogger perfLogger = SessionState.getPerfLogger();
protected ProcessorContext processorContext;
http://git-wip-us.apache.org/repos/asf/hive/blob/21861592/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
index 4a1a712..2d740ed 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
@@ -77,7 +77,7 @@ import org.json.JSONObject;
public class TezTask extends Task<TezWork> {
private static final String CLASS_NAME = TezTask.class.getName();
- private final PerfLogger perfLogger = PerfLogger.getPerfLogger();
+ private final PerfLogger perfLogger = SessionState.getPerfLogger();
private TezCounters counters;
http://git-wip-us.apache.org/repos/asf/hive/blob/21861592/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
index 11740d1..53bc1fa 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
@@ -25,11 +25,8 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedList;
import java.util.List;
import java.util.Map;
-import java.util.Queue;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
@@ -39,11 +36,9 @@ import java.util.concurrent.Future;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
-import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.Utilities;
@@ -52,19 +47,18 @@ import org.apache.hadoop.hive.ql.parse.SplitSample;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
import org.apache.hadoop.hive.ql.plan.PartitionDesc;
import org.apache.hadoop.hive.ql.plan.TableDesc;
+import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.shims.HadoopShims.CombineFileInputFormatShim;
import org.apache.hadoop.hive.shims.HadoopShimsSecure.InputSplitShim;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
-import org.apache.hadoop.io.compress.CompressionCodecFactory;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.InputFormat;
import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.hadoop.mapred.lib.CombineFileSplit;
@@ -462,7 +456,7 @@ public class CombineHiveInputFormat<K extends WritableComparable, V extends Writ
*/
@Override
public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException {
- PerfLogger perfLogger = PerfLogger.getPerfLogger();
+ PerfLogger perfLogger = SessionState.getPerfLogger();
perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.GET_SPLITS);
init(job);
http://git-wip-us.apache.org/repos/asf/hive/blob/21861592/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
index fd16b35..1ac1669 100755
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
@@ -37,7 +37,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil;
-import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.ql.exec.spark.SparkDynamicPartitionPruner;
import org.apache.hadoop.hive.ql.plan.TableDesc;
import org.apache.hadoop.hive.ql.exec.Operator;
@@ -46,10 +45,10 @@ import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.log.PerfLogger;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
import org.apache.hadoop.hive.ql.plan.MapWork;
-import org.apache.hadoop.hive.ql.plan.MergeJoinWork;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
import org.apache.hadoop.hive.ql.plan.PartitionDesc;
import org.apache.hadoop.hive.ql.plan.TableScanDesc;
+import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.io.Writable;
@@ -351,7 +350,7 @@ public class HiveInputFormat<K extends WritableComparable, V extends Writable>
}
public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException {
- PerfLogger perfLogger = PerfLogger.getPerfLogger();
+ PerfLogger perfLogger = SessionState.getPerfLogger();
perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.GET_SPLITS);
init(job);
Path[] dirs = getInputPaths(job);
http://git-wip-us.apache.org/repos/asf/hive/blob/21861592/ql/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java b/ql/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java
deleted file mode 100644
index 20ca195..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java
+++ /dev/null
@@ -1,195 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.log;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.ql.QueryPlan;
-import org.apache.hadoop.hive.ql.session.SessionState;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * PerfLogger.
- *
- * Can be used to measure and log the time spent by a piece of code.
- */
-public class PerfLogger {
- public static final String ACQUIRE_READ_WRITE_LOCKS = "acquireReadWriteLocks";
- public static final String COMPILE = "compile";
- public static final String PARSE = "parse";
- public static final String ANALYZE = "semanticAnalyze";
- public static final String DO_AUTHORIZATION = "doAuthorization";
- public static final String DRIVER_EXECUTE = "Driver.execute";
- public static final String INPUT_SUMMARY = "getInputSummary";
- public static final String GET_SPLITS = "getSplits";
- public static final String RUN_TASKS = "runTasks";
- public static final String SERIALIZE_PLAN = "serializePlan";
- public static final String DESERIALIZE_PLAN = "deserializePlan";
- public static final String CLONE_PLAN = "clonePlan";
- public static final String TASK = "task.";
- public static final String RELEASE_LOCKS = "releaseLocks";
- public static final String PRUNE_LISTING = "prune-listing";
- public static final String PARTITION_RETRIEVING = "partition-retrieving";
- public static final String PRE_HOOK = "PreHook.";
- public static final String POST_HOOK = "PostHook.";
- public static final String FAILURE_HOOK = "FailureHook.";
- public static final String DRIVER_RUN = "Driver.run";
- public static final String TIME_TO_SUBMIT = "TimeToSubmit";
- public static final String TEZ_SUBMIT_TO_RUNNING = "TezSubmitToRunningDag";
- public static final String TEZ_BUILD_DAG = "TezBuildDag";
- public static final String TEZ_SUBMIT_DAG = "TezSubmitDag";
- public static final String TEZ_RUN_DAG = "TezRunDag";
- public static final String TEZ_CREATE_VERTEX = "TezCreateVertex.";
- public static final String TEZ_RUN_VERTEX = "TezRunVertex.";
- public static final String TEZ_INITIALIZE_PROCESSOR = "TezInitializeProcessor";
- public static final String TEZ_RUN_PROCESSOR = "TezRunProcessor";
- public static final String TEZ_INIT_OPERATORS = "TezInitializeOperators";
- public static final String LOAD_HASHTABLE = "LoadHashtable";
-
- public static final String SPARK_SUBMIT_TO_RUNNING = "SparkSubmitToRunning";
- public static final String SPARK_BUILD_PLAN = "SparkBuildPlan";
- public static final String SPARK_BUILD_RDD_GRAPH = "SparkBuildRDDGraph";
- public static final String SPARK_SUBMIT_JOB = "SparkSubmitJob";
- public static final String SPARK_RUN_JOB = "SparkRunJob";
- public static final String SPARK_CREATE_TRAN = "SparkCreateTran.";
- public static final String SPARK_RUN_STAGE = "SparkRunStage.";
- public static final String SPARK_INIT_OPERATORS = "SparkInitializeOperators";
- public static final String SPARK_GENERATE_TASK_TREE = "SparkGenerateTaskTree";
- public static final String SPARK_OPTIMIZE_OPERATOR_TREE = "SparkOptimizeOperatorTree";
- public static final String SPARK_OPTIMIZE_TASK_TREE = "SparkOptimizeTaskTree";
- public static final String SPARK_FLUSH_HASHTABLE = "SparkFlushHashTable.";
-
- protected static final ThreadLocal<PerfLogger> perfLogger = new ThreadLocal<PerfLogger>();
-
- protected final Map<String, Long> startTimes = new HashMap<String, Long>();
- protected final Map<String, Long> endTimes = new HashMap<String, Long>();
-
- static final private Log LOG = LogFactory.getLog(PerfLogger.class.getName());
-
- public PerfLogger() {
- // Use getPerfLogger to get an instance of PerfLogger
- }
-
- public static PerfLogger getPerfLogger() {
- return getPerfLogger(false);
- }
-
- /**
- * Call this function to get an instance of PerfLogger.
- *
- * Use resetPerfLogger to require a new instance. Useful at the beginning of execution.
- *
- * @return Session perflogger if there's a sessionstate, otherwise return the thread local instance
- */
- public static PerfLogger getPerfLogger(boolean resetPerfLogger) {
- if (SessionState.get() == null) {
- if (perfLogger.get() == null || resetPerfLogger) {
- perfLogger.set(new PerfLogger());
- }
- return perfLogger.get();
- } else {
- return SessionState.get().getPerfLogger(resetPerfLogger);
- }
- }
-
- /**
- * Call this function when you start to measure time spent by a piece of code.
- * @param _log the logging object to be used.
- * @param method method or ID that identifies this perf log element.
- */
- public void PerfLogBegin(String callerName, String method) {
- long startTime = System.currentTimeMillis();
- LOG.info("<PERFLOG method=" + method + " from=" + callerName + ">");
- startTimes.put(method, new Long(startTime));
- }
-
- /**
- * Call this function in correspondence of PerfLogBegin to mark the end of the measurement.
- * @param _log
- * @param method
- * @return long duration the difference between now and startTime, or -1 if startTime is null
- */
- public long PerfLogEnd(String callerName, String method) {
- Long startTime = startTimes.get(method);
- long endTime = System.currentTimeMillis();
- long duration = -1;
-
- endTimes.put(method, new Long(endTime));
-
- StringBuilder sb = new StringBuilder("</PERFLOG method=").append(method);
- if (startTime != null) {
- sb.append(" start=").append(startTime);
- }
- sb.append(" end=").append(endTime);
- if (startTime != null) {
- duration = endTime - startTime.longValue();
- sb.append(" duration=").append(duration);
- }
- sb.append(" from=").append(callerName).append(">");
- LOG.info(sb);
-
- return duration;
- }
-
- /**
- * Call this function at the end of processing a query (any time after the last call to PerfLogEnd
- * for a given query) to run any cleanup/final steps that need to be run
- * @param _log
- */
- public void close(Log _log, QueryPlan queryPlan) {
-
- }
-
- public Long getStartTime(String method) {
- long startTime = 0L;
-
- if (startTimes.containsKey(method)) {
- startTime = startTimes.get(method);
- }
- return startTime;
- }
-
- public Long getEndTime(String method) {
- long endTime = 0L;
-
- if (endTimes.containsKey(method)) {
- endTime = endTimes.get(method);
- }
- return endTime;
- }
-
- public boolean startTimeHasMethod(String method) {
- return startTimes.containsKey(method);
- }
-
- public boolean endTimeHasMethod(String method) {
- return endTimes.containsKey(method);
- }
-
- public Long getDuration(String method) {
- long duration = 0;
- if (startTimes.containsKey(method) && endTimes.containsKey(method)) {
- duration = endTimes.get(method) - startTimes.get(method);
- }
- return duration;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/21861592/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java
index d264559..8eab603 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
+import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr;
@@ -400,7 +401,7 @@ public class PartitionPruner implements Transform {
// Now filter.
List<Partition> partitions = new ArrayList<Partition>();
boolean hasUnknownPartitions = false;
- PerfLogger perfLogger = PerfLogger.getPerfLogger();
+ PerfLogger perfLogger = SessionState.getPerfLogger();
if (!doEvalClientSide) {
perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.PARTITION_RETRIEVING);
try {
@@ -432,7 +433,7 @@ public class PartitionPruner implements Transform {
}
private static Set<Partition> getAllPartitions(Table tab) throws HiveException {
- PerfLogger perfLogger = PerfLogger.getPerfLogger();
+ PerfLogger perfLogger = SessionState.getPerfLogger();
perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.PARTITION_RETRIEVING);
Set<Partition> result = Hive.get().getAllPartitionsOf(tab);
perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.PARTITION_RETRIEVING);
@@ -450,7 +451,7 @@ public class PartitionPruner implements Transform {
*/
static private boolean pruneBySequentialScan(Table tab, List<Partition> partitions,
ExprNodeGenericFuncDesc prunerExpr, HiveConf conf) throws HiveException, MetaException {
- PerfLogger perfLogger = PerfLogger.getPerfLogger();
+ PerfLogger perfLogger = SessionState.getPerfLogger();
perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.PRUNE_LISTING);
List<String> partNames = Hive.get().getPartitionNames(
http://git-wip-us.apache.org/repos/asf/hive/blob/21861592/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java
index 27a1d99..9ec7fd6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java
@@ -87,6 +87,7 @@ import org.apache.hadoop.hive.ql.plan.MapWork;
import org.apache.hadoop.hive.ql.plan.MoveWork;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
import org.apache.hadoop.hive.ql.plan.SparkWork;
+import org.apache.hadoop.hive.ql.session.SessionState;
/**
* SparkCompiler translates the operator plan into SparkTasks.
@@ -95,7 +96,7 @@ import org.apache.hadoop.hive.ql.plan.SparkWork;
*/
public class SparkCompiler extends TaskCompiler {
private static final String CLASS_NAME = SparkCompiler.class.getName();
- private static final PerfLogger PERF_LOGGER = PerfLogger.getPerfLogger();
+ private static final PerfLogger PERF_LOGGER = SessionState.getPerfLogger();
private static final Log LOGGER = LogFactory.getLog(SparkCompiler.class);
public SparkCompiler() {
http://git-wip-us.apache.org/repos/asf/hive/blob/21861592/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
index 7ed8e5f..5f528167 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
@@ -104,6 +104,7 @@ public class SessionState {
private static final String LOCAL_SESSION_PATH_KEY = "_hive.local.session.path";
private static final String HDFS_SESSION_PATH_KEY = "_hive.hdfs.session.path";
private static final String TMP_TABLE_SPACE_KEY = "_hive.tmp_table_space";
+
private final Map<String, Map<String, Table>> tempTables = new HashMap<String, Map<String, Table>>();
private final Map<String, Map<String, ColumnStatisticsObj>> tempTableColStats =
new HashMap<String, Map<String, ColumnStatisticsObj>>();
@@ -596,7 +597,7 @@ public class SessionState {
* Create a given path if it doesn't exist.
*
* @param conf
- * @param pathString
+ * @param path
* @param permission
* @param isLocal
* @param isCleanUp
@@ -1523,25 +1524,37 @@ public class SessionState {
}
/**
- * @param resetPerfLogger
* @return Tries to return an instance of the class whose name is configured in
* hive.exec.perf.logger, but if it can't it just returns an instance of
* the base PerfLogger class
+ *
+ */
+ public static PerfLogger getPerfLogger() {
+ return getPerfLogger(false);
+ }
+ /**
+ * @param resetPerfLogger
+ * @return Tries to return an instance of the class whose name is configured in
+ * hive.exec.perf.logger, but if it can't it just returns an instance of
+ * the base PerfLogger class
+ *
*/
- public PerfLogger getPerfLogger(boolean resetPerfLogger) {
- if ((perfLogger == null) || resetPerfLogger) {
- try {
- perfLogger = (PerfLogger) ReflectionUtils.newInstance(conf.getClassByName(
- conf.getVar(ConfVars.HIVE_PERF_LOGGER)), conf);
- } catch (ClassNotFoundException e) {
- LOG.error("Performance Logger Class not found:" + e.getMessage());
- perfLogger = new PerfLogger();
- }
+ public static PerfLogger getPerfLogger(boolean resetPerfLogger) {
+ SessionState ss = get();
+ if (ss == null) {
+ return PerfLogger.getPerfLogger(null, resetPerfLogger);
+ } else if (ss.perfLogger != null && !resetPerfLogger) {
+ return ss.perfLogger;
+ } else {
+ PerfLogger perfLogger = PerfLogger.getPerfLogger(ss.getConf(), resetPerfLogger);
+ ss.perfLogger = perfLogger;
+ return perfLogger;
}
- return perfLogger;
}
+
+
public TezSessionState getTezSession() {
return tezSessionState;
}
[8/8] hive git commit: HIVE-11909 : LLAP: merge master into branch
(Sergey Shelukhin)
Posted by se...@apache.org.
HIVE-11909 : LLAP: merge master into branch (Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/47187618
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/47187618
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/47187618
Branch: refs/heads/llap
Commit: 47187618bbdd014c04465ca3eb4c7843132c9c25
Parents: 7148ea0 2a65989
Author: Sergey Shelukhin <se...@apache.org>
Authored: Mon Sep 21 16:04:08 2015 -0700
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Mon Sep 21 16:04:08 2015 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/log/PerfLogger.java | 196 +++++++++
.../HiveHBaseTableSnapshotInputFormat.java | 21 +-
.../queries/positive/hbase_handler_snapshot.q | 4 +
.../positive/hbase_handler_snapshot.q.out | 22 +
.../hive/thrift/TestHadoop20SAuthBridge.java | 420 ------------------
.../hive/thrift/TestHadoopAuthBridge23.java | 423 +++++++++++++++++++
.../hive/metastore/RetryingHMSHandler.java | 33 +-
.../java/org/apache/hadoop/hive/ql/Driver.java | 11 +-
.../hadoop/hive/ql/exec/MapJoinOperator.java | 3 +-
.../ql/exec/SparkHashTableSinkOperator.java | 3 +-
.../apache/hadoop/hive/ql/exec/Utilities.java | 11 +-
.../hadoop/hive/ql/exec/spark/SparkPlan.java | 3 +-
.../hive/ql/exec/spark/SparkPlanGenerator.java | 9 +-
.../hive/ql/exec/spark/SparkRecordHandler.java | 3 +-
.../hadoop/hive/ql/exec/spark/SparkTask.java | 2 +-
.../ql/exec/spark/status/SparkJobMonitor.java | 2 +-
.../hive/ql/exec/tez/RecordProcessor.java | 4 +-
.../hive/ql/exec/tez/ReduceRecordProcessor.java | 1 -
.../hive/ql/exec/tez/ReduceRecordSource.java | 3 +-
.../hadoop/hive/ql/exec/tez/TezJobMonitor.java | 17 +-
.../hadoop/hive/ql/exec/tez/TezProcessor.java | 3 +-
.../apache/hadoop/hive/ql/exec/tez/TezTask.java | 2 +-
.../hive/ql/io/CombineHiveInputFormat.java | 10 +-
.../hadoop/hive/ql/io/HiveInputFormat.java | 4 +-
.../apache/hadoop/hive/ql/log/PerfLogger.java | 195 ---------
.../hive/ql/optimizer/ppr/PartitionPruner.java | 7 +-
.../hive/ql/parse/spark/SparkCompiler.java | 3 +-
.../hadoop/hive/ql/session/SessionState.java | 37 +-
.../persistence/TestBytesBytesMultiHashMap.java | 3 +
.../ql/exec/persistence/TestHashPartition.java | 29 ++
.../clientpositive/parquet_ppd_boolean.q | 4 +-
.../queries/clientpositive/parquet_ppd_char.q | 12 +-
.../queries/clientpositive/parquet_ppd_date.q | 16 +-
.../clientpositive/parquet_ppd_decimal.q | 32 +-
.../clientpositive/parquet_ppd_timestamp.q | 16 +-
.../clientpositive/parquet_ppd_varchar.q | 12 +-
.../clientpositive/parquet_ppd_boolean.q.out | 28 +-
.../clientpositive/parquet_ppd_char.q.out | 84 ++--
.../clientpositive/parquet_ppd_date.q.out | 112 ++---
.../clientpositive/parquet_ppd_decimal.q.out | 224 +++++-----
.../clientpositive/parquet_ppd_timestamp.q.out | 112 ++---
.../clientpositive/parquet_ppd_varchar.q.out | 84 ++--
.../auth/LdapAuthenticationProviderImpl.java | 82 +++-
.../hadoop/hive/shims/Hadoop20SShims.java | 5 +-
.../apache/hadoop/hive/shims/Hadoop23Shims.java | 4 +-
.../hive/thrift/HadoopThriftAuthBridge.java | 3 +
46 files changed, 1263 insertions(+), 1051 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/47187618/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/47187618/ql/src/java/org/apache/hadoop/hive/ql/exec/SparkHashTableSinkOperator.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/47187618/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/47187618/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/47187618/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/RecordProcessor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/47187618/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java
index f177f0d,91ba2bb..7c41cb6
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java
@@@ -53,10 -51,7 +53,9 @@@ import org.apache.tez.runtime.api.Logic
import org.apache.tez.runtime.api.LogicalOutput;
import org.apache.tez.runtime.api.ProcessorContext;
import org.apache.tez.runtime.api.Reader;
- import org.apache.tez.runtime.library.api.KeyValuesReader;
+import com.google.common.collect.Lists;
+
/**
* Process input from tez LogicalInput and write output - for a map plan
* Just pump the records through the query plan.
http://git-wip-us.apache.org/repos/asf/hive/blob/47187618/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/47187618/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezProcessor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/47187618/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/47187618/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
index 3501d19,1ac1669..45ee9c5
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
@@@ -36,11 -36,7 +36,10 @@@ import org.apache.hadoop.conf.Configura
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil;
+import org.apache.hadoop.hive.llap.io.api.LlapIo;
+import org.apache.hadoop.hive.llap.io.api.LlapIoProxy;
- import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.ql.exec.spark.SparkDynamicPartitionPruner;
import org.apache.hadoop.hive.ql.plan.TableDesc;
import org.apache.hadoop.hive.ql.exec.Operator;
http://git-wip-us.apache.org/repos/asf/hive/blob/47187618/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
----------------------------------------------------------------------
diff --cc shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
index d47e3ff,93dcbd3..f60e8f0
--- a/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
+++ b/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
@@@ -68,11 -69,14 +68,13 @@@ import org.apache.hadoop.mapreduce.Task
import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.KerberosName;
-import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.tools.distcp2.DistCp;
import org.apache.hadoop.tools.distcp2.DistCpOptions;
+ import org.apache.hadoop.tools.distcp2.DistCpOptions.FileAttribute;
+
import org.apache.hadoop.util.Progressable;
-import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.VersionInfo;
http://git-wip-us.apache.org/repos/asf/hive/blob/47187618/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
----------------------------------------------------------------------
diff --cc shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
index fd6b17d,83369ee..5a136d8
--- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
+++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
@@@ -95,11 -92,12 +95,12 @@@ import org.apache.hadoop.mapreduce.Task
import org.apache.hadoop.mapreduce.task.JobContextImpl;
import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.authentication.util.KerberosName;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.util.KerberosName;
import org.apache.hadoop.tools.DistCp;
import org.apache.hadoop.tools.DistCpOptions;
+ import org.apache.hadoop.tools.DistCpOptions.FileAttribute;
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.tez.test.MiniTezCluster;
[7/8] hive git commit: HIVE-11849: NPE in
HiveHBaseTableShapshotInputFormat in query with just count(*) (Enis Soztutar
via Jason Dere)
Posted by se...@apache.org.
HIVE-11849: NPE in HiveHBaseTableShapshotInputFormat in query with just count(*) (Enis Soztutar via Jason Dere)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2a65989a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2a65989a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2a65989a
Branch: refs/heads/llap
Commit: 2a65989a48e043404b7060296be8da9d3494911e
Parents: 4ff5b25
Author: Jason Dere <jd...@hortonworks.com>
Authored: Mon Sep 21 10:59:21 2015 -0700
Committer: Jason Dere <jd...@hortonworks.com>
Committed: Mon Sep 21 10:59:21 2015 -0700
----------------------------------------------------------------------
.../HiveHBaseTableSnapshotInputFormat.java | 21 ++++++++++++-------
.../queries/positive/hbase_handler_snapshot.q | 4 ++++
.../positive/hbase_handler_snapshot.q.out | 22 ++++++++++++++++++++
3 files changed, 39 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/2a65989a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableSnapshotInputFormat.java
----------------------------------------------------------------------
diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableSnapshotInputFormat.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableSnapshotInputFormat.java
index 45e4de9..aa3a02f 100644
--- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableSnapshotInputFormat.java
+++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableSnapshotInputFormat.java
@@ -24,6 +24,9 @@ import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapred.TableInputFormat;
import org.apache.hadoop.hbase.mapred.TableSnapshotInputFormat;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
+import org.apache.hadoop.hbase.util.Base64;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.InputFormat;
@@ -41,15 +44,17 @@ public class HiveHBaseTableSnapshotInputFormat
TableSnapshotInputFormat delegate = new TableSnapshotInputFormat();
private static void setColumns(JobConf job) throws IOException {
- // hbase mapred API doesn't support scan at the moment.
Scan scan = HiveHBaseInputFormatUtil.getScan(job);
- byte[][] families = scan.getFamilies();
- StringBuilder sb = new StringBuilder();
- for (int i = 0; i < families.length; i++) {
- if (i > 0) sb.append(" ");
- sb.append(Bytes.toString(families[i]));
- }
- job.set(TableInputFormat.COLUMN_LIST, sb.toString());
+ job.set(org.apache.hadoop.hbase.mapreduce.TableInputFormat.SCAN,
+ convertScanToString(scan));
+ }
+
+ // TODO: Once HBASE-11163 is completed, use that API, or switch to
+ // using mapreduce version of the APIs. rather than mapred
+ // Copied from HBase's TableMapreduceUtil since it is not public API
+ static String convertScanToString(Scan scan) throws IOException {
+ ClientProtos.Scan proto = ProtobufUtil.toScan(scan);
+ return Base64.encodeBytes(proto.toByteArray());
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/2a65989a/hbase-handler/src/test/queries/positive/hbase_handler_snapshot.q
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/queries/positive/hbase_handler_snapshot.q b/hbase-handler/src/test/queries/positive/hbase_handler_snapshot.q
index 11d52fd..ebdc63c 100644
--- a/hbase-handler/src/test/queries/positive/hbase_handler_snapshot.q
+++ b/hbase-handler/src/test/queries/positive/hbase_handler_snapshot.q
@@ -2,3 +2,7 @@ SET hive.hbase.snapshot.name=src_hbase_snapshot;
SET hive.hbase.snapshot.restoredir=/tmp;
SELECT * FROM src_hbase LIMIT 5;
+
+SELECT value FROM src_hbase LIMIT 5;
+
+select count(*) from src_hbase;
http://git-wip-us.apache.org/repos/asf/hive/blob/2a65989a/hbase-handler/src/test/results/positive/hbase_handler_snapshot.q.out
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/results/positive/hbase_handler_snapshot.q.out b/hbase-handler/src/test/results/positive/hbase_handler_snapshot.q.out
index 1cb18b2..731646c 100644
--- a/hbase-handler/src/test/results/positive/hbase_handler_snapshot.q.out
+++ b/hbase-handler/src/test/results/positive/hbase_handler_snapshot.q.out
@@ -11,3 +11,25 @@ POSTHOOK: Input: default@src_hbase
100 val_100
103 val_103
104 val_104
+PREHOOK: query: SELECT value FROM src_hbase LIMIT 5
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_hbase
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT value FROM src_hbase LIMIT 5
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_hbase
+#### A masked pattern was here ####
+val_0
+val_10
+val_100
+val_103
+val_104
+PREHOOK: query: select count(*) from src_hbase
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_hbase
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from src_hbase
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_hbase
+#### A masked pattern was here ####
+309
[6/8] hive git commit: HIVE-11889: Add unit test for HIVE-11449 (Wei
Zheng via Jason Dere)
Posted by se...@apache.org.
HIVE-11889: Add unit test for HIVE-11449 (Wei Zheng via Jason Dere)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4ff5b258
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4ff5b258
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4ff5b258
Branch: refs/heads/llap
Commit: 4ff5b258c93a1996f320cab19c58465a2aab38bc
Parents: 262bae6
Author: Jason Dere <jd...@hortonworks.com>
Authored: Mon Sep 21 10:45:37 2015 -0700
Committer: Jason Dere <jd...@hortonworks.com>
Committed: Mon Sep 21 10:45:37 2015 -0700
----------------------------------------------------------------------
.../persistence/TestBytesBytesMultiHashMap.java | 3 ++
.../ql/exec/persistence/TestHashPartition.java | 29 ++++++++++++++++++++
2 files changed, 32 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/4ff5b258/ql/src/test/org/apache/hadoop/hive/ql/exec/persistence/TestBytesBytesMultiHashMap.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/persistence/TestBytesBytesMultiHashMap.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/persistence/TestBytesBytesMultiHashMap.java
index c417b6f..aed9214 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/persistence/TestBytesBytesMultiHashMap.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/persistence/TestBytesBytesMultiHashMap.java
@@ -43,6 +43,9 @@ public class TestBytesBytesMultiHashMap {
assertEquals(CAPACITY, map.getCapacity());
map = new BytesBytesMultiHashMap(9, LOAD_FACTOR, WB_SIZE);
assertEquals(16, map.getCapacity());
+
+ // Verify the scenario when maxProbeSize is a very small value, it doesn't fail
+ BytesBytesMultiHashMap map1 = new BytesBytesMultiHashMap(1024, (float) 0.75, 524288, 1);
}
@Test
http://git-wip-us.apache.org/repos/asf/hive/blob/4ff5b258/ql/src/test/org/apache/hadoop/hive/ql/exec/persistence/TestHashPartition.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/persistence/TestHashPartition.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/persistence/TestHashPartition.java
new file mode 100644
index 0000000..a6e52bd
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/persistence/TestHashPartition.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.exec.persistence;
+
+import org.apache.hadoop.hive.ql.exec.persistence.HybridHashTableContainer.HashPartition;
+import org.junit.Test;
+
+public class TestHashPartition {
+
+ @Test
+ public void testHashPartition() throws Exception {
+ HashPartition hashPartition = new HashPartition(1024, (float) 0.75, 524288, 1, true);
+ }
+}
[5/8] hive git commit: HIVE-11826: 'hadoop.proxyuser.hive.groups'
configuration doesn't prevent unauthorized user to access metastore
Posted by se...@apache.org.
HIVE-11826: 'hadoop.proxyuser.hive.groups' configuration doesn't prevent unauthorized user to access metastore
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/262bae6b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/262bae6b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/262bae6b
Branch: refs/heads/llap
Commit: 262bae6b1e93abba6947e31785e6174b072904dc
Parents: 92b42ae
Author: Aihua Xu <ai...@gmail.com>
Authored: Mon Sep 21 09:44:57 2015 -0700
Committer: Chao Sun <su...@apache.org>
Committed: Mon Sep 21 09:44:57 2015 -0700
----------------------------------------------------------------------
.../hive/thrift/TestHadoop20SAuthBridge.java | 420 ------------------
.../hive/thrift/TestHadoopAuthBridge23.java | 423 +++++++++++++++++++
.../hive/thrift/HadoopThriftAuthBridge.java | 3 +
3 files changed, 426 insertions(+), 420 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/262bae6b/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/thrift/TestHadoop20SAuthBridge.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/thrift/TestHadoop20SAuthBridge.java b/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/thrift/TestHadoop20SAuthBridge.java
deleted file mode 100644
index f6029b1..0000000
--- a/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/thrift/TestHadoop20SAuthBridge.java
+++ /dev/null
@@ -1,420 +0,0 @@
-package org.apache.hadoop.hive.thrift;
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-import java.io.ByteArrayInputStream;
-import java.io.DataInputStream;
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.NetworkInterface;
-import java.net.ServerSocket;
-import java.security.PrivilegedExceptionAction;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Enumeration;
-import java.util.List;
-import java.util.Map;
-
-import junit.framework.TestCase;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.HiveMetaStore;
-import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
-import org.apache.hadoop.hive.metastore.MetaStoreUtils;
-import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge.Server.ServerMode;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.SaslRpcServer;
-import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
-import org.apache.hadoop.security.authorize.AuthorizationException;
-import org.apache.hadoop.security.authorize.DefaultImpersonationProvider;
-import org.apache.hadoop.security.authorize.ProxyUsers;
-import org.apache.hadoop.security.token.SecretManager.InvalidToken;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager.DelegationTokenInformation;
-import org.apache.hadoop.security.token.delegation.DelegationKey;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.thrift.transport.TSaslServerTransport;
-import org.apache.thrift.transport.TTransportException;
-import org.apache.thrift.transport.TTransportFactory;
-
-public class TestHadoop20SAuthBridge extends TestCase {
-
- /**
- * set to true when metastore token manager has intitialized token manager
- * through call to HadoopThriftAuthBridge20S.Server.startDelegationTokenSecretManager
- */
- static volatile boolean isMetastoreTokenManagerInited;
-
- private static class MyHadoopThriftAuthBridge20S extends HadoopThriftAuthBridge {
- @Override
- public Server createServer(String keytabFile, String principalConf)
- throws TTransportException {
- //Create a Server that doesn't interpret any Kerberos stuff
- return new Server();
- }
-
- static class Server extends HadoopThriftAuthBridge.Server {
- public Server() throws TTransportException {
- super();
- }
- @Override
- public TTransportFactory createTransportFactory(Map<String, String> saslProps)
- throws TTransportException {
- TSaslServerTransport.Factory transFactory =
- new TSaslServerTransport.Factory();
- transFactory.addServerDefinition(AuthMethod.DIGEST.getMechanismName(),
- null, SaslRpcServer.SASL_DEFAULT_REALM,
- saslProps,
- new SaslDigestCallbackHandler(secretManager));
-
- return new TUGIAssumingTransportFactory(transFactory, realUgi);
- }
- static DelegationTokenStore TOKEN_STORE = new MemoryTokenStore();
-
- @Override
- protected DelegationTokenStore getTokenStore(Configuration conf) throws IOException {
- return TOKEN_STORE;
- }
-
- @Override
- public void startDelegationTokenSecretManager(Configuration conf, Object hms, ServerMode sm)
- throws IOException{
- super.startDelegationTokenSecretManager(conf, hms, sm);
- isMetastoreTokenManagerInited = true;
- }
-
- }
- }
-
-
- private HiveConf conf;
-
- private void configureSuperUserIPAddresses(Configuration conf,
- String superUserShortName) throws IOException {
- List<String> ipList = new ArrayList<String>();
- Enumeration<NetworkInterface> netInterfaceList = NetworkInterface
- .getNetworkInterfaces();
- while (netInterfaceList.hasMoreElements()) {
- NetworkInterface inf = netInterfaceList.nextElement();
- Enumeration<InetAddress> addrList = inf.getInetAddresses();
- while (addrList.hasMoreElements()) {
- InetAddress addr = addrList.nextElement();
- ipList.add(addr.getHostAddress());
- }
- }
- StringBuilder builder = new StringBuilder();
- for (String ip : ipList) {
- builder.append(ip);
- builder.append(',');
- }
- builder.append("127.0.1.1,");
- builder.append(InetAddress.getLocalHost().getCanonicalHostName());
- conf.setStrings(DefaultImpersonationProvider.getTestProvider().getProxySuperuserIpConfKey(superUserShortName),
- builder.toString());
- }
-
- public void setup() throws Exception {
- isMetastoreTokenManagerInited = false;
- int port = findFreePort();
- System.setProperty(HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL.varname,
- "true");
- System.setProperty(HiveConf.ConfVars.METASTOREURIS.varname,
- "thrift://localhost:" + port);
- System.setProperty(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, new Path(
- System.getProperty("test.build.data", "/tmp")).toString());
- conf = new HiveConf(TestHadoop20SAuthBridge.class);
- MetaStoreUtils.startMetaStore(port, new MyHadoopThriftAuthBridge20S());
- }
-
- /**
- * Test delegation token store/load from shared store.
- * @throws Exception
- */
- public void testDelegationTokenSharedStore() throws Exception {
- UserGroupInformation clientUgi = UserGroupInformation.getCurrentUser();
-
- TokenStoreDelegationTokenSecretManager tokenManager =
- new TokenStoreDelegationTokenSecretManager(0, 60*60*1000, 60*60*1000, 0,
- MyHadoopThriftAuthBridge20S.Server.TOKEN_STORE);
- // initializes current key
- tokenManager.startThreads();
- tokenManager.stopThreads();
-
- String tokenStrForm = tokenManager.getDelegationToken(clientUgi.getShortUserName());
- Token<DelegationTokenIdentifier> t= new Token<DelegationTokenIdentifier>();
- t.decodeFromUrlString(tokenStrForm);
-
- //check whether the username in the token is what we expect
- DelegationTokenIdentifier d = new DelegationTokenIdentifier();
- d.readFields(new DataInputStream(new ByteArrayInputStream(
- t.getIdentifier())));
- assertTrue("Usernames don't match",
- clientUgi.getShortUserName().equals(d.getUser().getShortUserName()));
-
- DelegationTokenInformation tokenInfo = MyHadoopThriftAuthBridge20S.Server.TOKEN_STORE
- .getToken(d);
- assertNotNull("token not in store", tokenInfo);
- assertFalse("duplicate token add",
- MyHadoopThriftAuthBridge20S.Server.TOKEN_STORE.addToken(d, tokenInfo));
-
- // check keys are copied from token store when token is loaded
- TokenStoreDelegationTokenSecretManager anotherManager =
- new TokenStoreDelegationTokenSecretManager(0, 0, 0, 0,
- MyHadoopThriftAuthBridge20S.Server.TOKEN_STORE);
- assertEquals("master keys empty on init", 0,
- anotherManager.getAllKeys().length);
- assertNotNull("token loaded",
- anotherManager.retrievePassword(d));
- anotherManager.renewToken(t, clientUgi.getShortUserName());
- assertEquals("master keys not loaded from store",
- MyHadoopThriftAuthBridge20S.Server.TOKEN_STORE.getMasterKeys().length,
- anotherManager.getAllKeys().length);
-
- // cancel the delegation token
- tokenManager.cancelDelegationToken(tokenStrForm);
- assertNull("token not removed from store after cancel",
- MyHadoopThriftAuthBridge20S.Server.TOKEN_STORE.getToken(d));
- assertFalse("token removed (again)",
- MyHadoopThriftAuthBridge20S.Server.TOKEN_STORE.removeToken(d));
- try {
- anotherManager.retrievePassword(d);
- fail("InvalidToken expected after cancel");
- } catch (InvalidToken ex) {
- // expected
- }
-
- // token expiration
- MyHadoopThriftAuthBridge20S.Server.TOKEN_STORE.addToken(d,
- new DelegationTokenInformation(0, t.getPassword()));
- assertNotNull(MyHadoopThriftAuthBridge20S.Server.TOKEN_STORE.getToken(d));
- anotherManager.removeExpiredTokens();
- assertNull("Expired token not removed",
- MyHadoopThriftAuthBridge20S.Server.TOKEN_STORE.getToken(d));
-
- // key expiration - create an already expired key
- anotherManager.startThreads(); // generates initial key
- anotherManager.stopThreads();
- DelegationKey expiredKey = new DelegationKey(-1, 0, anotherManager.getAllKeys()[0].getKey());
- anotherManager.logUpdateMasterKey(expiredKey); // updates key with sequence number
- assertTrue("expired key not in allKeys",
- anotherManager.reloadKeys().containsKey(expiredKey.getKeyId()));
- anotherManager.rollMasterKeyExt();
- assertFalse("Expired key not removed",
- anotherManager.reloadKeys().containsKey(expiredKey.getKeyId()));
- }
-
- public void testSaslWithHiveMetaStore() throws Exception {
- setup();
- UserGroupInformation clientUgi = UserGroupInformation.getCurrentUser();
- obtainTokenAndAddIntoUGI(clientUgi, null);
- obtainTokenAndAddIntoUGI(clientUgi, "tokenForFooTablePartition");
- }
-
- public void testMetastoreProxyUser() throws Exception {
- setup();
-
- final String proxyUserName = "proxyUser";
- //set the configuration up such that proxyUser can act on
- //behalf of all users belonging to the group foo_bar_group (
- //a dummy group)
- String[] groupNames =
- new String[] { "foo_bar_group" };
- setGroupsInConf(groupNames, proxyUserName);
-
- final UserGroupInformation delegationTokenUser =
- UserGroupInformation.getCurrentUser();
-
- final UserGroupInformation proxyUserUgi =
- UserGroupInformation.createRemoteUser(proxyUserName);
- String tokenStrForm = proxyUserUgi.doAs(new PrivilegedExceptionAction<String>() {
- public String run() throws Exception {
- try {
- //Since the user running the test won't belong to a non-existent group
- //foo_bar_group, the call to getDelegationTokenStr will fail
- return getDelegationTokenStr(delegationTokenUser, proxyUserUgi);
- } catch (AuthorizationException ae) {
- return null;
- }
- }
- });
- assertTrue("Expected the getDelegationToken call to fail",
- tokenStrForm == null);
-
- //set the configuration up such that proxyUser can act on
- //behalf of all users belonging to the real group(s) that the
- //user running the test belongs to
- setGroupsInConf(UserGroupInformation.getCurrentUser().getGroupNames(),
- proxyUserName);
- tokenStrForm = proxyUserUgi.doAs(new PrivilegedExceptionAction<String>() {
- public String run() throws Exception {
- try {
- //Since the user running the test belongs to the group
- //obtained above the call to getDelegationTokenStr will succeed
- return getDelegationTokenStr(delegationTokenUser, proxyUserUgi);
- } catch (AuthorizationException ae) {
- return null;
- }
- }
- });
- assertTrue("Expected the getDelegationToken call to not fail",
- tokenStrForm != null);
- Token<DelegationTokenIdentifier> t= new Token<DelegationTokenIdentifier>();
- t.decodeFromUrlString(tokenStrForm);
- //check whether the username in the token is what we expect
- DelegationTokenIdentifier d = new DelegationTokenIdentifier();
- d.readFields(new DataInputStream(new ByteArrayInputStream(
- t.getIdentifier())));
- assertTrue("Usernames don't match",
- delegationTokenUser.getShortUserName().equals(d.getUser().getShortUserName()));
-
- }
-
- private void setGroupsInConf(String[] groupNames, String proxyUserName)
- throws IOException {
- conf.set(
- DefaultImpersonationProvider.getTestProvider().getProxySuperuserGroupConfKey(proxyUserName),
- StringUtils.join(",", Arrays.asList(groupNames)));
- configureSuperUserIPAddresses(conf, proxyUserName);
- ProxyUsers.refreshSuperUserGroupsConfiguration(conf);
- }
-
- private String getDelegationTokenStr(UserGroupInformation ownerUgi,
- UserGroupInformation realUgi) throws Exception {
- //obtain a token by directly invoking the metastore operation(without going
- //through the thrift interface). Obtaining a token makes the secret manager
- //aware of the user and that it gave the token to the user
- //also set the authentication method explicitly to KERBEROS. Since the
- //metastore checks whether the authentication method is KERBEROS or not
- //for getDelegationToken, and the testcases don't use
- //kerberos, this needs to be done
-
- waitForMetastoreTokenInit();
-
- HadoopThriftAuthBridge.Server.authenticationMethod
- .set(AuthenticationMethod.KERBEROS);
- HadoopThriftAuthBridge.Server.remoteAddress.set(InetAddress.getLocalHost());
- return
- HiveMetaStore.getDelegationToken(ownerUgi.getShortUserName(),
- realUgi.getShortUserName());
- }
-
- /**
- * Wait for metastore to have initialized token manager
- * This does not have to be done in other metastore test cases as they
- * use metastore client which will retry few times on failure
- * @throws InterruptedException
- */
- private void waitForMetastoreTokenInit() throws InterruptedException {
- int waitAttempts = 30;
- while(waitAttempts > 0 && !isMetastoreTokenManagerInited){
- Thread.sleep(1000);
- waitAttempts--;
- }
- }
-
- private void obtainTokenAndAddIntoUGI(UserGroupInformation clientUgi,
- String tokenSig) throws Exception {
- String tokenStrForm = getDelegationTokenStr(clientUgi, clientUgi);
- Token<DelegationTokenIdentifier> t= new Token<DelegationTokenIdentifier>();
- t.decodeFromUrlString(tokenStrForm);
-
- //check whether the username in the token is what we expect
- DelegationTokenIdentifier d = new DelegationTokenIdentifier();
- d.readFields(new DataInputStream(new ByteArrayInputStream(
- t.getIdentifier())));
- assertTrue("Usernames don't match",
- clientUgi.getShortUserName().equals(d.getUser().getShortUserName()));
-
- if (tokenSig != null) {
- conf.set("hive.metastore.token.signature", tokenSig);
- t.setService(new Text(tokenSig));
- }
- //add the token to the clientUgi for securely talking to the metastore
- clientUgi.addToken(t);
- //Create the metastore client as the clientUgi. Doing so this
- //way will give the client access to the token that was added earlier
- //in the clientUgi
- HiveMetaStoreClient hiveClient =
- clientUgi.doAs(new PrivilegedExceptionAction<HiveMetaStoreClient>() {
- public HiveMetaStoreClient run() throws Exception {
- HiveMetaStoreClient hiveClient =
- new HiveMetaStoreClient(conf);
- return hiveClient;
- }
- });
-
- assertTrue("Couldn't connect to metastore", hiveClient != null);
-
- //try out some metastore operations
- createDBAndVerifyExistence(hiveClient);
-
- //check that getDelegationToken fails since we are not authenticating
- //over kerberos
- boolean pass = false;
- try {
- hiveClient.getDelegationToken(clientUgi.getUserName());
- } catch (MetaException ex) {
- pass = true;
- }
- assertTrue("Expected the getDelegationToken call to fail", pass == true);
- hiveClient.close();
-
- //Now cancel the delegation token
- HiveMetaStore.cancelDelegationToken(tokenStrForm);
-
- //now metastore connection should fail
- hiveClient =
- clientUgi.doAs(new PrivilegedExceptionAction<HiveMetaStoreClient>() {
- public HiveMetaStoreClient run() {
- try {
- HiveMetaStoreClient hiveClient =
- new HiveMetaStoreClient(conf);
- return hiveClient;
- } catch (MetaException e) {
- return null;
- }
- }
- });
- assertTrue("Expected metastore operations to fail", hiveClient == null);
- }
-
- private void createDBAndVerifyExistence(HiveMetaStoreClient client)
- throws Exception {
- String dbName = "simpdb";
- Database db = new Database();
- db.setName(dbName);
- client.createDatabase(db);
- Database db1 = client.getDatabase(dbName);
- client.dropDatabase(dbName);
- assertTrue("Databases do not match", db1.getName().equals(db.getName()));
- }
-
- private int findFreePort() throws IOException {
- ServerSocket socket= new ServerSocket(0);
- int port = socket.getLocalPort();
- socket.close();
- return port;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/262bae6b/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/thrift/TestHadoopAuthBridge23.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/thrift/TestHadoopAuthBridge23.java b/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/thrift/TestHadoopAuthBridge23.java
new file mode 100644
index 0000000..40b161a
--- /dev/null
+++ b/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/thrift/TestHadoopAuthBridge23.java
@@ -0,0 +1,423 @@
+package org.apache.hadoop.hive.thrift;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.NetworkInterface;
+import java.net.ServerSocket;
+import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Enumeration;
+import java.util.List;
+import java.util.Map;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.HiveMetaStore;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.MetaStoreUtils;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.SaslRpcServer;
+import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
+import org.apache.hadoop.security.authorize.AuthorizationException;
+import org.apache.hadoop.security.authorize.DefaultImpersonationProvider;
+import org.apache.hadoop.security.authorize.ProxyUsers;
+import org.apache.hadoop.security.token.SecretManager.InvalidToken;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager.DelegationTokenInformation;
+import org.apache.hadoop.security.token.delegation.DelegationKey;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.thrift.transport.TSaslServerTransport;
+import org.apache.thrift.transport.TTransportException;
+import org.apache.thrift.transport.TTransportFactory;
+
+public class TestHadoopAuthBridge23 extends TestCase {
+
+ /**
+ * set to true when metastore token manager has intitialized token manager
+ * through call to HadoopThriftAuthBridge23.Server.startDelegationTokenSecretManager
+ */
+ static volatile boolean isMetastoreTokenManagerInited;
+
+ private static class MyHadoopThriftAuthBridge23 extends HadoopThriftAuthBridge23 {
+ @Override
+ public Server createServer(String keytabFile, String principalConf)
+ throws TTransportException {
+ //Create a Server that doesn't interpret any Kerberos stuff
+ return new Server();
+ }
+
+ static class Server extends HadoopThriftAuthBridge.Server {
+ public Server() throws TTransportException {
+ super();
+ }
+ @Override
+ public TTransportFactory createTransportFactory(Map<String, String> saslProps)
+ throws TTransportException {
+ TSaslServerTransport.Factory transFactory =
+ new TSaslServerTransport.Factory();
+ transFactory.addServerDefinition(AuthMethod.DIGEST.getMechanismName(),
+ null, SaslRpcServer.SASL_DEFAULT_REALM,
+ saslProps,
+ new SaslDigestCallbackHandler(secretManager));
+
+ return new TUGIAssumingTransportFactory(transFactory, realUgi);
+ }
+ static DelegationTokenStore TOKEN_STORE = new MemoryTokenStore();
+
+ @Override
+ protected DelegationTokenStore getTokenStore(Configuration conf) throws IOException {
+ return TOKEN_STORE;
+ }
+
+ @Override
+ public void startDelegationTokenSecretManager(Configuration conf, Object hms, ServerMode sm)
+ throws IOException{
+ super.startDelegationTokenSecretManager(conf, hms, sm);
+ isMetastoreTokenManagerInited = true;
+ }
+
+ }
+ }
+
+
+ private HiveConf conf;
+
+ private void configureSuperUserIPAddresses(Configuration conf,
+ String superUserShortName) throws IOException {
+ List<String> ipList = new ArrayList<String>();
+ Enumeration<NetworkInterface> netInterfaceList = NetworkInterface
+ .getNetworkInterfaces();
+ while (netInterfaceList.hasMoreElements()) {
+ NetworkInterface inf = netInterfaceList.nextElement();
+ Enumeration<InetAddress> addrList = inf.getInetAddresses();
+ while (addrList.hasMoreElements()) {
+ InetAddress addr = addrList.nextElement();
+ ipList.add(addr.getHostAddress());
+ }
+ }
+ StringBuilder builder = new StringBuilder();
+ for (String ip : ipList) {
+ builder.append(ip);
+ builder.append(',');
+ }
+ builder.append("127.0.1.1,");
+ builder.append(InetAddress.getLocalHost().getCanonicalHostName());
+ conf.setStrings(DefaultImpersonationProvider.getTestProvider().getProxySuperuserIpConfKey(superUserShortName),
+ builder.toString());
+ }
+
+ public void setup() throws Exception {
+ isMetastoreTokenManagerInited = false;
+ int port = findFreePort();
+ System.setProperty(HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL.varname,
+ "true");
+ System.setProperty(HiveConf.ConfVars.METASTOREURIS.varname,
+ "thrift://localhost:" + port);
+ System.setProperty(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, new Path(
+ System.getProperty("test.build.data", "/tmp")).toString());
+ conf = new HiveConf(TestHadoopAuthBridge23.class);
+ MetaStoreUtils.startMetaStore(port, new MyHadoopThriftAuthBridge23());
+ }
+
+ /**
+ * Test delegation token store/load from shared store.
+ * @throws Exception
+ */
+ public void testDelegationTokenSharedStore() throws Exception {
+ UserGroupInformation clientUgi = UserGroupInformation.getCurrentUser();
+
+ TokenStoreDelegationTokenSecretManager tokenManager =
+ new TokenStoreDelegationTokenSecretManager(0, 60*60*1000, 60*60*1000, 0,
+ MyHadoopThriftAuthBridge23.Server.TOKEN_STORE);
+ // initializes current key
+ tokenManager.startThreads();
+ tokenManager.stopThreads();
+
+ String tokenStrForm = tokenManager.getDelegationToken(clientUgi.getShortUserName());
+ Token<DelegationTokenIdentifier> t= new Token<DelegationTokenIdentifier>();
+ t.decodeFromUrlString(tokenStrForm);
+
+ //check whether the username in the token is what we expect
+ DelegationTokenIdentifier d = new DelegationTokenIdentifier();
+ d.readFields(new DataInputStream(new ByteArrayInputStream(
+ t.getIdentifier())));
+ assertTrue("Usernames don't match",
+ clientUgi.getShortUserName().equals(d.getUser().getShortUserName()));
+
+ DelegationTokenInformation tokenInfo = MyHadoopThriftAuthBridge23.Server.TOKEN_STORE
+ .getToken(d);
+ assertNotNull("token not in store", tokenInfo);
+ assertFalse("duplicate token add",
+ MyHadoopThriftAuthBridge23.Server.TOKEN_STORE.addToken(d, tokenInfo));
+
+ // check keys are copied from token store when token is loaded
+ TokenStoreDelegationTokenSecretManager anotherManager =
+ new TokenStoreDelegationTokenSecretManager(0, 0, 0, 0,
+ MyHadoopThriftAuthBridge23.Server.TOKEN_STORE);
+ assertEquals("master keys empty on init", 0,
+ anotherManager.getAllKeys().length);
+ assertNotNull("token loaded",
+ anotherManager.retrievePassword(d));
+ anotherManager.renewToken(t, clientUgi.getShortUserName());
+ assertEquals("master keys not loaded from store",
+ MyHadoopThriftAuthBridge23.Server.TOKEN_STORE.getMasterKeys().length,
+ anotherManager.getAllKeys().length);
+
+ // cancel the delegation token
+ tokenManager.cancelDelegationToken(tokenStrForm);
+ assertNull("token not removed from store after cancel",
+ MyHadoopThriftAuthBridge23.Server.TOKEN_STORE.getToken(d));
+ assertFalse("token removed (again)",
+ MyHadoopThriftAuthBridge23.Server.TOKEN_STORE.removeToken(d));
+ try {
+ anotherManager.retrievePassword(d);
+ fail("InvalidToken expected after cancel");
+ } catch (InvalidToken ex) {
+ // expected
+ }
+
+ // token expiration
+ MyHadoopThriftAuthBridge23.Server.TOKEN_STORE.addToken(d,
+ new DelegationTokenInformation(0, t.getPassword()));
+ assertNotNull(MyHadoopThriftAuthBridge23.Server.TOKEN_STORE.getToken(d));
+ anotherManager.removeExpiredTokens();
+ assertNull("Expired token not removed",
+ MyHadoopThriftAuthBridge23.Server.TOKEN_STORE.getToken(d));
+
+ // key expiration - create an already expired key
+ anotherManager.startThreads(); // generates initial key
+ anotherManager.stopThreads();
+ DelegationKey expiredKey = new DelegationKey(-1, 0, anotherManager.getAllKeys()[0].getKey());
+ anotherManager.logUpdateMasterKey(expiredKey); // updates key with sequence number
+ assertTrue("expired key not in allKeys",
+ anotherManager.reloadKeys().containsKey(expiredKey.getKeyId()));
+ anotherManager.rollMasterKeyExt();
+ assertFalse("Expired key not removed",
+ anotherManager.reloadKeys().containsKey(expiredKey.getKeyId()));
+ }
+
+ public void testSaslWithHiveMetaStore() throws Exception {
+ setup();
+
+ final String proxyUserName = UserGroupInformation.getCurrentUser().getShortUserName();
+ setGroupsInConf(new String[] {"*"}, proxyUserName);
+
+ UserGroupInformation clientUgi = UserGroupInformation.getCurrentUser();
+ obtainTokenAndAddIntoUGI(clientUgi, null);
+ obtainTokenAndAddIntoUGI(clientUgi, "tokenForFooTablePartition");
+ }
+
+ public void testMetastoreProxyUser() throws Exception {
+ setup();
+
+ final String proxyUserName = "proxyUser";
+ //set the configuration up such that proxyUser can act on
+ //behalf of all users belonging to the group foo_bar_group (
+ //a dummy group)
+ String[] groupNames =
+ new String[] { "foo_bar_group" };
+ setGroupsInConf(groupNames, proxyUserName);
+
+ final UserGroupInformation delegationTokenUser =
+ UserGroupInformation.getCurrentUser();
+
+ final UserGroupInformation proxyUserUgi =
+ UserGroupInformation.createRemoteUser(proxyUserName);
+ String tokenStrForm = proxyUserUgi.doAs(new PrivilegedExceptionAction<String>() {
+ public String run() throws Exception {
+ try {
+ //Since the user running the test won't belong to a non-existent group
+ //foo_bar_group, the call to getDelegationTokenStr will fail
+ return getDelegationTokenStr(delegationTokenUser, proxyUserUgi);
+ } catch (AuthorizationException ae) {
+ return null;
+ }
+ }
+ });
+ assertTrue("Expected the getDelegationToken call to fail",
+ tokenStrForm == null);
+
+ //set the configuration up such that proxyUser can act on
+ //behalf of all users belonging to the real group(s) that the
+ //user running the test belongs to
+ setGroupsInConf(UserGroupInformation.getCurrentUser().getGroupNames(),
+ proxyUserName);
+ tokenStrForm = proxyUserUgi.doAs(new PrivilegedExceptionAction<String>() {
+ public String run() throws Exception {
+ try {
+ //Since the user running the test belongs to the group
+ //obtained above the call to getDelegationTokenStr will succeed
+ return getDelegationTokenStr(delegationTokenUser, proxyUserUgi);
+ } catch (AuthorizationException ae) {
+ return null;
+ }
+ }
+ });
+ assertTrue("Expected the getDelegationToken call to not fail",
+ tokenStrForm != null);
+ Token<DelegationTokenIdentifier> t= new Token<DelegationTokenIdentifier>();
+ t.decodeFromUrlString(tokenStrForm);
+ //check whether the username in the token is what we expect
+ DelegationTokenIdentifier d = new DelegationTokenIdentifier();
+ d.readFields(new DataInputStream(new ByteArrayInputStream(
+ t.getIdentifier())));
+ assertTrue("Usernames don't match",
+ delegationTokenUser.getShortUserName().equals(d.getUser().getShortUserName()));
+
+ }
+
+ private void setGroupsInConf(String[] groupNames, String proxyUserName)
+ throws IOException {
+ conf.set(
+ DefaultImpersonationProvider.getTestProvider().getProxySuperuserGroupConfKey(proxyUserName),
+ StringUtils.join(",", Arrays.asList(groupNames)));
+ configureSuperUserIPAddresses(conf, proxyUserName);
+ ProxyUsers.refreshSuperUserGroupsConfiguration(conf);
+ }
+
+ private String getDelegationTokenStr(UserGroupInformation ownerUgi,
+ UserGroupInformation realUgi) throws Exception {
+ //obtain a token by directly invoking the metastore operation(without going
+ //through the thrift interface). Obtaining a token makes the secret manager
+ //aware of the user and that it gave the token to the user
+ //also set the authentication method explicitly to KERBEROS. Since the
+ //metastore checks whether the authentication method is KERBEROS or not
+ //for getDelegationToken, and the testcases don't use
+ //kerberos, this needs to be done
+
+ waitForMetastoreTokenInit();
+
+ HadoopThriftAuthBridge.Server.authenticationMethod
+ .set(AuthenticationMethod.KERBEROS);
+ HadoopThriftAuthBridge.Server.remoteAddress.set(InetAddress.getLocalHost());
+ return
+ HiveMetaStore.getDelegationToken(ownerUgi.getShortUserName(),
+ realUgi.getShortUserName());
+ }
+
+ /**
+ * Wait for metastore to have initialized token manager
+ * This does not have to be done in other metastore test cases as they
+ * use metastore client which will retry few times on failure
+ * @throws InterruptedException
+ */
+ private void waitForMetastoreTokenInit() throws InterruptedException {
+ int waitAttempts = 30;
+ while(waitAttempts > 0 && !isMetastoreTokenManagerInited){
+ Thread.sleep(1000);
+ waitAttempts--;
+ }
+ }
+
+ private void obtainTokenAndAddIntoUGI(UserGroupInformation clientUgi,
+ String tokenSig) throws Exception {
+ String tokenStrForm = getDelegationTokenStr(clientUgi, clientUgi);
+ Token<DelegationTokenIdentifier> t= new Token<DelegationTokenIdentifier>();
+ t.decodeFromUrlString(tokenStrForm);
+
+ //check whether the username in the token is what we expect
+ DelegationTokenIdentifier d = new DelegationTokenIdentifier();
+ d.readFields(new DataInputStream(new ByteArrayInputStream(
+ t.getIdentifier())));
+ assertTrue("Usernames don't match",
+ clientUgi.getShortUserName().equals(d.getUser().getShortUserName()));
+
+ if (tokenSig != null) {
+ conf.set("hive.metastore.token.signature", tokenSig);
+ t.setService(new Text(tokenSig));
+ }
+ //add the token to the clientUgi for securely talking to the metastore
+ clientUgi.addToken(t);
+ //Create the metastore client as the clientUgi. Doing so this
+ //way will give the client access to the token that was added earlier
+ //in the clientUgi
+ HiveMetaStoreClient hiveClient =
+ clientUgi.doAs(new PrivilegedExceptionAction<HiveMetaStoreClient>() {
+ public HiveMetaStoreClient run() throws Exception {
+ HiveMetaStoreClient hiveClient =
+ new HiveMetaStoreClient(conf);
+ return hiveClient;
+ }
+ });
+
+ assertTrue("Couldn't connect to metastore", hiveClient != null);
+
+ //try out some metastore operations
+ createDBAndVerifyExistence(hiveClient);
+
+ //check that getDelegationToken fails since we are not authenticating
+ //over kerberos
+ boolean pass = false;
+ try {
+ hiveClient.getDelegationToken(clientUgi.getUserName());
+ } catch (MetaException ex) {
+ pass = true;
+ }
+ assertTrue("Expected the getDelegationToken call to fail", pass == true);
+ hiveClient.close();
+
+ //Now cancel the delegation token
+ HiveMetaStore.cancelDelegationToken(tokenStrForm);
+
+ //now metastore connection should fail
+ hiveClient =
+ clientUgi.doAs(new PrivilegedExceptionAction<HiveMetaStoreClient>() {
+ public HiveMetaStoreClient run() {
+ try {
+ HiveMetaStoreClient hiveClient =
+ new HiveMetaStoreClient(conf);
+ return hiveClient;
+ } catch (MetaException e) {
+ return null;
+ }
+ }
+ });
+ assertTrue("Expected metastore operations to fail", hiveClient == null);
+ }
+
+ private void createDBAndVerifyExistence(HiveMetaStoreClient client)
+ throws Exception {
+ String dbName = "simpdb";
+ Database db = new Database();
+ db.setName(dbName);
+ client.createDatabase(db);
+ Database db1 = client.getDatabase(dbName);
+ client.dropDatabase(dbName);
+ assertTrue("Databases do not match", db1.getName().equals(db.getName()));
+ }
+
+ private int findFreePort() throws IOException {
+ ServerSocket socket= new ServerSocket(0);
+ int port = socket.getLocalPort();
+ socket.close();
+ return port;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/262bae6b/shims/common/src/main/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge.java
----------------------------------------------------------------------
diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge.java b/shims/common/src/main/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge.java
index 9d49ad5..7ed7265 100644
--- a/shims/common/src/main/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge.java
+++ b/shims/common/src/main/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge.java
@@ -671,6 +671,9 @@ public class HadoopThriftAuthBridge {
if (useProxy) {
clientUgi = UserGroupInformation.createProxyUser(
endUser, UserGroupInformation.getLoginUser());
+
+ ProxyUsers.authorize(clientUgi, getRemoteAddress().getHostAddress(), null);
+
remoteUser.set(clientUgi.getShortUserName());
LOG.debug("Set remoteUser :" + remoteUser.get());
return clientUgi.doAs(new PrivilegedExceptionAction<Boolean>() {