You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jd...@apache.org on 2016/04/04 22:36:36 UTC
[01/50] [abbrv] hive git commit: HIVE-13295: Improvement to LDAP
search queries in HS2 LDAP Authenticator (Naveen Gangam via Chaoyu Tang)
Repository: hive
Updated Branches:
refs/heads/llap 28d1082b4 -> a7b0ca733
HIVE-13295: Improvement to LDAP search queries in HS2 LDAP Authenticator (Naveen Gangam via Chaoyu Tang)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e665f020
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e665f020
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e665f020
Branch: refs/heads/llap
Commit: e665f020b419cf9096006c45f4afcda13fa9e882
Parents: 55383d8
Author: ctang <ct...@cloudera.com>
Authored: Thu Mar 24 09:34:59 2016 -0700
Committer: ctang <ct...@cloudera.com>
Committed: Thu Mar 24 09:34:59 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 9 +
.../auth/LdapAuthenticationProviderImpl.java | 317 ++++++++++---------
.../auth/TestLdapAtnProviderWithMiniDS.java | 200 +++++++++++-
3 files changed, 373 insertions(+), 153 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/e665f020/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index b8b9dcf..b8870f2 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2237,6 +2237,15 @@ public class HiveConf extends Configuration {
HIVE_SERVER2_PLAIN_LDAP_USERFILTER("hive.server2.authentication.ldap.userFilter", null,
"COMMA-separated list of LDAP usernames (just short names, not full DNs).\n" +
"For example: hiveuser,impalauser,hiveadmin,hadoopadmin"),
+ HIVE_SERVER2_PLAIN_LDAP_GUIDKEY("hive.server2.authentication.ldap.guidKey", "uid",
+ "LDAP attribute name whose values are unique in this LDAP server.\n" +
+ "For example: uid or CN."),
+ HIVE_SERVER2_PLAIN_LDAP_GROUPMEMBERSHIP_KEY("hive.server2.authentication.ldap.groupMembershipKey", "member",
+ "LDAP attribute name on the user entry that references a group, the user belongs to.\n" +
+ "For example: member, uniqueMember or memberUid"),
+ HIVE_SERVER2_PLAIN_LDAP_GROUPCLASS_KEY("hive.server2.authentication.ldap.groupClassKey", "groupOfNames",
+ "LDAP attribute name on the group entry that is to be used in LDAP group searches.\n" +
+ "For example: group, groupOfNames or groupOfUniqueNames."),
HIVE_SERVER2_PLAIN_LDAP_CUSTOMLDAPQUERY("hive.server2.authentication.ldap.customLDAPQuery", null,
"A full LDAP query that LDAP Atn provider uses to execute against LDAP Server.\n" +
"If this query returns a null resultset, the LDAP Provider fails the Authentication\n" +
http://git-wip-us.apache.org/repos/asf/hive/blob/e665f020/service/src/java/org/apache/hive/service/auth/LdapAuthenticationProviderImpl.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/auth/LdapAuthenticationProviderImpl.java b/service/src/java/org/apache/hive/service/auth/LdapAuthenticationProviderImpl.java
index 9b0b14d..8f64672 100644
--- a/service/src/java/org/apache/hive/service/auth/LdapAuthenticationProviderImpl.java
+++ b/service/src/java/org/apache/hive/service/auth/LdapAuthenticationProviderImpl.java
@@ -41,7 +41,6 @@ import org.slf4j.LoggerFactory;
public class LdapAuthenticationProviderImpl implements PasswdAuthenticationProvider {
private static final Logger LOG = LoggerFactory.getLogger(LdapAuthenticationProviderImpl.class);
- private static final String DN_ATTR = "distinguishedName";
private String ldapURL;
private String baseDN;
@@ -51,6 +50,9 @@ public class LdapAuthenticationProviderImpl implements PasswdAuthenticationProvi
private static List<String> userFilter;
private static List<String> groupFilter;
private String customQuery;
+ private static String guid_attr;
+ private static String groupMembership_attr;
+ private static String groupClass_attr;
LdapAuthenticationProviderImpl(HiveConf conf) {
init(conf);
@@ -61,65 +63,66 @@ public class LdapAuthenticationProviderImpl implements PasswdAuthenticationProvi
baseDN = conf.getVar(HiveConf.ConfVars.HIVE_SERVER2_PLAIN_LDAP_BASEDN);
ldapDomain = conf.getVar(HiveConf.ConfVars.HIVE_SERVER2_PLAIN_LDAP_DOMAIN);
customQuery = conf.getVar(HiveConf.ConfVars.HIVE_SERVER2_PLAIN_LDAP_CUSTOMLDAPQUERY);
-
- if (customQuery == null) {
- groupBases = new ArrayList<String>();
- userBases = new ArrayList<String>();
- String groupDNPatterns = conf.getVar(HiveConf.ConfVars.HIVE_SERVER2_PLAIN_LDAP_GROUPDNPATTERN);
- String groupFilterVal = conf.getVar(HiveConf.ConfVars.HIVE_SERVER2_PLAIN_LDAP_GROUPFILTER);
- String userDNPatterns = conf.getVar(HiveConf.ConfVars.HIVE_SERVER2_PLAIN_LDAP_USERDNPATTERN);
- String userFilterVal = conf.getVar(HiveConf.ConfVars.HIVE_SERVER2_PLAIN_LDAP_USERFILTER);
-
- // parse COLON delimited root DNs for users/groups that may or may not be under BaseDN.
- // Expect the root DNs be fully qualified including the baseDN
- if (groupDNPatterns != null && groupDNPatterns.trim().length() > 0) {
- String[] groupTokens = groupDNPatterns.split(":");
- for (int i = 0; i < groupTokens.length; i++) {
- if (groupTokens[i].contains(",") && groupTokens[i].contains("=")) {
- groupBases.add(groupTokens[i]);
- } else {
- LOG.warn("Unexpected format for " + HiveConf.ConfVars.HIVE_SERVER2_PLAIN_LDAP_GROUPDNPATTERN
- + "..ignoring " + groupTokens[i]);
- }
+ guid_attr = conf.getVar(HiveConf.ConfVars.HIVE_SERVER2_PLAIN_LDAP_GUIDKEY);
+ groupBases = new ArrayList<String>();
+ userBases = new ArrayList<String>();
+ userFilter = new ArrayList<String>();
+ groupFilter = new ArrayList<String>();
+
+ String groupDNPatterns = conf.getVar(HiveConf.ConfVars.HIVE_SERVER2_PLAIN_LDAP_GROUPDNPATTERN);
+ String groupFilterVal = conf.getVar(HiveConf.ConfVars.HIVE_SERVER2_PLAIN_LDAP_GROUPFILTER);
+ String userDNPatterns = conf.getVar(HiveConf.ConfVars.HIVE_SERVER2_PLAIN_LDAP_USERDNPATTERN);
+ String userFilterVal = conf.getVar(HiveConf.ConfVars.HIVE_SERVER2_PLAIN_LDAP_USERFILTER);
+ groupMembership_attr = conf.getVar(HiveConf.ConfVars.HIVE_SERVER2_PLAIN_LDAP_GROUPMEMBERSHIP_KEY);
+ groupClass_attr = conf.getVar(HiveConf.ConfVars.HIVE_SERVER2_PLAIN_LDAP_GROUPCLASS_KEY);
+
+ // parse COLON delimited root DNs for users/groups that may or may not be under BaseDN.
+ // Expect the root DNs be fully qualified including the baseDN
+ if (groupDNPatterns != null && groupDNPatterns.trim().length() > 0) {
+ String[] groupTokens = groupDNPatterns.split(":");
+ for (int i = 0; i < groupTokens.length; i++) {
+ if (groupTokens[i].contains(",") && groupTokens[i].contains("=")) {
+ groupBases.add(groupTokens[i]);
+ } else {
+ LOG.warn("Unexpected format for " + HiveConf.ConfVars.HIVE_SERVER2_PLAIN_LDAP_GROUPDNPATTERN
+ + "..ignoring " + groupTokens[i]);
}
- } else if (baseDN != null) {
- groupBases.add("uid=%s," + baseDN);
}
+ } else if (baseDN != null) {
+ groupBases.add(guid_attr + "=%s," + baseDN);
+ }
- if (groupFilterVal != null && groupFilterVal.trim().length() > 0) {
- groupFilter = new ArrayList<String>();
- String[] groups = groupFilterVal.split(",");
- for (int i = 0; i < groups.length; i++) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Filtered group: " + groups[i]);
- }
- groupFilter.add(groups[i]);
+ if (groupFilterVal != null && groupFilterVal.trim().length() > 0) {
+ String[] groups = groupFilterVal.split(",");
+ for (int i = 0; i < groups.length; i++) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Filtered group: " + groups[i]);
}
+ groupFilter.add(groups[i]);
}
+ }
- if (userDNPatterns != null && userDNPatterns.trim().length() > 0) {
- String[] userTokens = userDNPatterns.split(":");
- for (int i = 0; i < userTokens.length; i++) {
- if (userTokens[i].contains(",") && userTokens[i].contains("=")) {
- userBases.add(userTokens[i]);
- } else {
- LOG.warn("Unexpected format for " + HiveConf.ConfVars.HIVE_SERVER2_PLAIN_LDAP_USERDNPATTERN
- + "..ignoring " + userTokens[i]);
- }
+ if (userDNPatterns != null && userDNPatterns.trim().length() > 0) {
+ String[] userTokens = userDNPatterns.split(":");
+ for (int i = 0; i < userTokens.length; i++) {
+ if (userTokens[i].contains(",") && userTokens[i].contains("=")) {
+ userBases.add(userTokens[i]);
+ } else {
+ LOG.warn("Unexpected format for " + HiveConf.ConfVars.HIVE_SERVER2_PLAIN_LDAP_USERDNPATTERN
+ + "..ignoring " + userTokens[i]);
}
- } else if (baseDN != null) {
- userBases.add("uid=%s," + baseDN);
}
+ } else if (baseDN != null) {
+ userBases.add(guid_attr + "=%s," + baseDN);
+ }
- if (userFilterVal != null && userFilterVal.trim().length() > 0) {
- userFilter = new ArrayList<String>();
- String[] users = userFilterVal.split(",");
- for (int i = 0; i < users.length; i++) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Filtered user: " + users[i]);
- }
- userFilter.add(users[i]);
+ if (userFilterVal != null && userFilterVal.trim().length() > 0) {
+ String[] users = userFilterVal.split(",");
+ for (int i = 0; i < users.length; i++) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Filtered user: " + users[i]);
}
+ userFilter.add(users[i]);
}
}
}
@@ -159,7 +162,7 @@ public class LdapAuthenticationProviderImpl implements PasswdAuthenticationProvi
try {
bindDN = listIter.next().replaceAll("%s", user);
env.put(Context.SECURITY_PRINCIPAL, bindDN);
- LOG.debug("Connecting using principal=" + user + " at url=" + ldapURL);
+ LOG.debug("Connecting using DN " + bindDN + " at url " + ldapURL);
ctx = new InitialDirContext(env);
break;
} catch (NamingException e) {
@@ -168,7 +171,7 @@ public class LdapAuthenticationProviderImpl implements PasswdAuthenticationProvi
}
} else {
env.put(Context.SECURITY_PRINCIPAL, user);
- LOG.debug("Connecting using principal=" + user + " at url=" + ldapURL);
+ LOG.debug("Connecting using principal " + user + " at url " + ldapURL);
try {
ctx = new InitialDirContext(env);
} catch (NamingException e) {
@@ -177,9 +180,11 @@ public class LdapAuthenticationProviderImpl implements PasswdAuthenticationProvi
}
if (ctx == null) {
+ LOG.debug("Could not connect to the LDAP Server:Authentication failed for " + user);
throw new AuthenticationException("LDAP Authentication failed for user", ex);
}
+ LOG.debug("Connected using principal=" + user + " at url=" + ldapURL);
try {
if (isDN(user) || hasDomain(user)) {
userName = extractName(user);
@@ -187,7 +192,24 @@ public class LdapAuthenticationProviderImpl implements PasswdAuthenticationProvi
userName = user;
}
- if (userFilter == null && groupFilter == null && customQuery == null && userBases.size() > 0) {
+ // if a custom LDAP query is specified, it takes precedence over other configuration properties.
+ // if the user being authenticated is part of the resultset from the custom query, it succeeds.
+ if (customQuery != null) {
+ List<String> resultList = executeLDAPQuery(ctx, customQuery, baseDN);
+ if (resultList != null) {
+ for (String matchedDN : resultList) {
+ LOG.info("<queried user=" + matchedDN.split(",",2)[0].split("=",2)[1] + ",user=" + user + ">");
+ if (matchedDN.split(",",2)[0].split("=",2)[1].equalsIgnoreCase(user) ||
+ matchedDN.equalsIgnoreCase(user)) {
+ LOG.info("Authentication succeeded based on result set from LDAP query");
+ return;
+ }
+ }
+ }
+ LOG.info("Authentication failed based on result set from custom LDAP query");
+ throw new AuthenticationException("Authentication failed: LDAP query " +
+ "from property returned no data");
+ } else if (userBases.size() > 0) {
if (isDN(user)) {
userDN = findUserDNByDN(ctx, user);
} else {
@@ -196,7 +218,7 @@ public class LdapAuthenticationProviderImpl implements PasswdAuthenticationProvi
}
if (userDN == null) {
- userDN = findUserDNByName(ctx, baseDN, userName);
+ userDN = findUserDNByName(ctx, userName);
}
}
@@ -205,86 +227,60 @@ public class LdapAuthenticationProviderImpl implements PasswdAuthenticationProvi
if (userDN == null) {
throw new AuthenticationException("Authentication failed: User search failed");
}
- return;
- }
-
- if (customQuery != null) {
- List<String> resultList = executeLDAPQuery(ctx, customQuery, baseDN);
- if (resultList != null) {
- for (String matchedDN : resultList) {
- if (matchedDN.split(",",2)[0].split("=",2)[1].equalsIgnoreCase(user)) {
- LOG.info("Authentication succeeded based on result set from LDAP query");
- return;
- }
- }
- }
- throw new AuthenticationException("Authentication failed: LDAP query " +
- "from property returned no data");
- }
- // This section checks if the user satisfies the specified user filter.
- if (userFilter != null && userFilter.size() > 0) {
- LOG.info("Authenticating user " + user + " using user filter");
+ // This section checks if the user satisfies the specified user filter.
+ if (userFilter.size() > 0) {
+ LOG.info("Authenticating user " + user + " using user filter");
- boolean success = false;
- for (String filteredUser : userFilter) {
- if (filteredUser.equalsIgnoreCase(userName)) {
- LOG.debug("User filter partially satisfied");
- success = true;
- break;
+ if (userDN != null) {
+ LOG.info("User filter partially satisfied");
}
- }
-
- if (!success) {
- LOG.info("Authentication failed based on user membership");
- throw new AuthenticationException("Authentication failed: User not a member " +
- "of specified list");
- }
-
- userDN = findUserDNByPattern(ctx, userName);
- if (userDN != null) {
- LOG.info("User filter entirely satisfied");
- } else {
- LOG.info("User " + user + " could not be found in the configured UserBaseDN," +
- "authentication failed");
- throw new AuthenticationException("Authentication failed: UserDN could not be " +
- "found in specified User base(s)");
- }
- }
- if (groupFilter != null && groupFilter.size() > 0) {
- LOG.debug("Authenticating user " + user + " using group membership:");
+ boolean success = false;
+ for (String filteredUser : userFilter) {
+ if (filteredUser.equalsIgnoreCase(userName)) {
+ LOG.debug("User filter entirely satisfied");
+ success = true;
+ break;
+ }
+ }
- // if only groupFilter is configured.
- if (userDN == null) {
- userDN = findUserDNByName(ctx, baseDN, userName);
+ if (!success) {
+ LOG.info("Authentication failed based on user membership");
+ throw new AuthenticationException("Authentication failed: User not a member " +
+ "of specified list");
+ }
}
- List<String> userGroups = getGroupsForUser(ctx, userDN);
- if (LOG.isDebugEnabled()) {
- LOG.debug("User member of :");
- prettyPrint(userGroups);
- }
+ // This section checks if the user satisfies the specified user filter.
+ if (groupFilter.size() > 0) {
+ LOG.debug("Authenticating user " + user + " using group membership");
+ List<String> userGroups = getGroupsForUser(ctx, userDN);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("User member of :");
+ prettyPrint(userGroups);
+ }
- if (userGroups != null) {
- for (String elem : userGroups) {
- String shortName = ((elem.split(","))[0].split("="))[1];
- String groupDN = elem.split(",", 2)[1];
- LOG.debug("Checking group:DN=" + elem + ",shortName=" + shortName +
- ",groupDN=" + groupDN);
- if (groupFilter.contains(shortName)) {
- LOG.info("Authentication succeeded based on group membership");
- return;
+ if (userGroups != null) {
+ for (String elem : userGroups) {
+ String shortName = ((elem.split(","))[0].split("="))[1];
+ if (groupFilter.contains(shortName)) {
+ LOG.info("Authentication succeeded based on group membership");
+ return;
+ }
}
}
- }
- throw new AuthenticationException("Authentication failed: User not a member of " +
- "listed groups");
+ LOG.debug("Authentication failed: User is not a member of configured groups");
+ throw new AuthenticationException("Authentication failed: User not a member of " +
+ "listed groups");
+ }
+ LOG.info("Authentication succeeded using ldap user search");
+ return;
}
-
+ // Ideally we should not be here. Indicates partially configured LDAP Service.
+ // We allow it for now for backward compatibility.
LOG.info("Simple password authentication succeeded");
-
} catch (NamingException e) {
throw new AuthenticationException("LDAP Authentication failed for user", e);
} finally {
@@ -337,7 +333,7 @@ public class LdapAuthenticationProviderImpl implements PasswdAuthenticationProvi
*/
public static String findGroupDNByName(DirContext ctx, String baseDN, String groupName)
throws NamingException {
- String searchFilter = "(&(objectClass=group)(CN=" + groupName + "))";
+ String searchFilter = "(&(objectClass=" + groupClass_attr + ")(" + guid_attr + "=" + groupName + "))";
List<String> results = null;
results = findDNByName(ctx, baseDN, searchFilter, 2);
@@ -410,9 +406,9 @@ public class LdapAuthenticationProviderImpl implements PasswdAuthenticationProvi
* @param userName A unique userid that is to be located in the LDAP.
* @return LDAP DN if the user is found in LDAP, null otherwise.
*/
- public static String findUserDNByName(DirContext ctx, String baseDN, String userName)
+ public static String findUserDNByName(DirContext ctx, String userName)
throws NamingException {
- if (baseDN == null) {
+ if (userBases.size() == 0) {
return null;
}
@@ -421,23 +417,28 @@ public class LdapAuthenticationProviderImpl implements PasswdAuthenticationProvi
"(|(uid=" + userName + ")(sAMAccountName=" + userName + ")))",
"(|(cn=*" + userName + "*)))"
};
- String searchFilter = null;
- List<String> results = null;
+
+ String searchFilter = null;
+ List<String> results = null;
+ ListIterator<String> listIter = userBases.listIterator();
for (int i = 0; i < suffix.length; i++) {
searchFilter = baseFilter + suffix[i];
- results = findDNByName(ctx, baseDN, searchFilter, 2);
- if(results == null) {
- continue;
- }
+ while (listIter.hasNext()) {
+ results = findDNByName(ctx, listIter.next().split(",",2)[1], searchFilter, 2);
- if(results != null && results.size() > 1) {
- //make sure there is not another item available, there should be only 1 match
- LOG.info("Matched multiple users for the user: " + userName + ",returning null");
- return null;
+ if(results == null) {
+ continue;
+ }
+
+ if(results != null && results.size() > 1) {
+ //make sure there is not another item available, there should be only 1 match
+ LOG.info("Matched multiple users for the user: " + userName + ",returning null");
+ return null;
+ }
+ return results.get(0);
}
- return results.get(0);
}
return null;
}
@@ -525,37 +526,47 @@ public class LdapAuthenticationProviderImpl implements PasswdAuthenticationProvi
/**
* This helper method finds all the groups a given user belongs to.
- * This method relies on the "memberOf" attribute being set on the user that references
- * the group the group. The returned list ONLY includes direct groups the user belongs to.
- * Parent groups of these direct groups are NOT included.
+ * This method relies on the attribute,configurable via HIVE_SERVER2_PLAIN_LDAP_GROUPMEMBERSHIP_KEY,
+ * being set on the user entry that references the group. The returned list ONLY includes direct
+ * groups the user belongs to. Parent groups of these direct groups are NOT included.
* @param ctx DirContext for the LDAP Connection.
- * @param userName A unique userid that is to be located in the LDAP.
+ * @param userDN A unique userDN that is to be located in the LDAP.
* @return List of Group DNs the user belongs to, emptylist otherwise.
*/
public static List<String> getGroupsForUser(DirContext ctx, String userDN)
throws NamingException {
List<String> groupList = new ArrayList<String>();
- String searchFilter = "(" + DN_ATTR + "=" + userDN + ")";
+ String user = extractName(userDN);
+ String searchFilter = "(&(objectClass=" + groupClass_attr + ")(|(" +
+ groupMembership_attr + "=" + userDN + ")(" +
+ groupMembership_attr + "=" + user + ")))";
SearchControls searchControls = new SearchControls();
+ NamingEnumeration<SearchResult> results = null;
+ SearchResult result = null;
+ String groupBase = null;
LOG.debug("getGroupsForUser:searchFilter=" + searchFilter);
- String[] attrIDs = { "memberOf" };
+ String[] attrIDs = new String[0];
searchControls.setSearchScope(SearchControls.SUBTREE_SCOPE);
searchControls.setReturningAttributes(attrIDs);
- // treat everything after the first COMMA as a baseDN for the search to find this user
- NamingEnumeration<SearchResult> results = ctx.search(userDN.split(",",2)[1], searchFilter,
- searchControls);
- while(results.hasMoreElements()) {
- NamingEnumeration<? extends Attribute> groups = results.next().getAttributes().getAll();
- while (groups.hasMore()) {
- Attribute attr = groups.next();
- NamingEnumeration<?> list = attr.getAll();
- while (list.hasMore()) {
- groupList.add((String)list.next());
+ ListIterator<String> listIter = groupBases.listIterator();
+ while (listIter.hasNext()) {
+ try {
+ groupBase = listIter.next().split(",", 2)[1];
+ LOG.debug("Searching for groups under " + groupBase);
+ results = ctx.search(groupBase, searchFilter, searchControls);
+
+ while(results.hasMoreElements()) {
+ result = results.nextElement();
+ LOG.debug("Found Group:" + result.getNameInNamespace());
+ groupList.add(result.getNameInNamespace());
}
+ } catch (NamingException e) {
+ LOG.warn("Exception searching for user groups", e);
}
}
+
return groupList;
}
@@ -577,6 +588,10 @@ public class LdapAuthenticationProviderImpl implements PasswdAuthenticationProvi
*/
public static List<String> executeLDAPQuery(DirContext ctx, String query, String rootDN)
throws NamingException {
+ if (rootDN == null) {
+ return null;
+ }
+
SearchControls searchControls = new SearchControls();
List<String> list = new ArrayList<String>();
String[] returnAttributes = new String[0]; //empty set
http://git-wip-us.apache.org/repos/asf/hive/blob/e665f020/service/src/test/org/apache/hive/service/auth/TestLdapAtnProviderWithMiniDS.java
----------------------------------------------------------------------
diff --git a/service/src/test/org/apache/hive/service/auth/TestLdapAtnProviderWithMiniDS.java b/service/src/test/org/apache/hive/service/auth/TestLdapAtnProviderWithMiniDS.java
index 832ebdf..ee9262a 100644
--- a/service/src/test/org/apache/hive/service/auth/TestLdapAtnProviderWithMiniDS.java
+++ b/service/src/test/org/apache/hive/service/auth/TestLdapAtnProviderWithMiniDS.java
@@ -109,21 +109,23 @@ partitions = {
"dn: uid=group1,ou=Groups,dc=example,dc=com",
"distinguishedName: uid=group1,ou=Groups,dc=example,dc=com",
"objectClass: top",
- "objectClass: organizationalUnit",
+ "objectClass: groupOfNames",
"objectClass: ExtensibleObject",
"cn: group1",
"ou: Groups",
"sn: group1",
+ "member: uid=user1,ou=People,dc=example,dc=com",
"dn: uid=group2,ou=Groups,dc=example,dc=com",
"distinguishedName: uid=group2,ou=Groups,dc=example,dc=com",
"objectClass: top",
- "objectClass: organizationalUnit",
+ "objectClass: groupOfNames",
"objectClass: ExtensibleObject",
"givenName: Group2",
"ou: Groups",
"cn: group1",
"sn: group1",
+ "member: uid=user2,ou=People,dc=example,dc=com",
"dn: uid=user1,ou=People,dc=example,dc=com",
"distinguishedName: uid=user1,ou=People,dc=example,dc=com",
@@ -535,4 +537,198 @@ public class TestLdapAtnProviderWithMiniDS extends AbstractLdapTestUnit {
}
}
+ @Test
+ public void testUserFilterPositive() throws Exception {
+ String user;
+ Map<String, String> ldapProperties = new HashMap<String, String>();
+ ldapProperties.put("hive.server2.authentication.ldap.userDNPattern", "uid=%s,ou=People,dc=example,dc=com");
+ ldapProperties.put("hive.server2.authentication.ldap.userFilter", "user2");
+ initLdapAtn(ldapProperties);
+
+ user = "uid=user2,ou=People,dc=example,dc=com";
+ try {
+ ldapProvider.Authenticate(user, "user2");
+ assertTrue("testUserFilterPositive: Authentication succeeded for " + user + " as expected", true);
+
+ user = "user2";
+ ldapProvider.Authenticate(user, "user2");
+ assertTrue("testUserFilterPositive: Authentication succeeded for " + user + " as expected", true);
+ } catch (AuthenticationException e) {
+ Assert.fail("testUserFilterPositive: Authentication failed for " + user + ",user expected to pass userfilter");
+ }
+
+ ldapProperties = new HashMap<String, String>();
+ ldapProperties.put("hive.server2.authentication.ldap.userDNPattern", "uid=%s,ou=People,dc=example,dc=com");
+ ldapProperties.put("hive.server2.authentication.ldap.userFilter", "user1");
+ initLdapAtn(ldapProperties);
+
+ try {
+ user = "uid=user1,ou=People,dc=example,dc=com";
+ ldapProvider.Authenticate(user, "user1");
+ assertTrue("testUserFilterPositive: Authentication succeeded for " + user + " as expected", true);
+
+ user = "user1";
+ ldapProvider.Authenticate(user, "user1");
+ assertTrue("testUserFilterPositive: Authentication succeeded for " + user + " as expected", true);
+ } catch (AuthenticationException e) {
+ Assert.fail("testUserFilterPositive: Authentication failed for " + user + ",user expected to pass userfilter");
+ }
+
+ ldapProperties = new HashMap<String, String>();
+ ldapProperties.put("hive.server2.authentication.ldap.userDNPattern", "uid=%s,ou=People,dc=example,dc=com");
+ ldapProperties.put("hive.server2.authentication.ldap.userFilter", "user2,user1");
+ initLdapAtn(ldapProperties);
+
+ try {
+ user = "uid=user1,ou=People,dc=example,dc=com";
+ ldapProvider.Authenticate(user, "user1");
+ assertTrue("testUserFilterPositive: Authentication succeeded for " + user + " as expected", true);
+
+ user = "user2";
+ ldapProvider.Authenticate(user, "user2");
+ assertTrue("testUserFilterPositive: Authentication succeeded for " + user + " as expected", true);
+
+ } catch (AuthenticationException e) {
+ Assert.fail("testUserFilterPositive: Authentication failed for user, user is expected to pass userfilter");
+ }
+ }
+
+ @Test
+ public void testUserFilterNegative() throws Exception {
+ String user;
+ Map<String, String> ldapProperties = new HashMap<String, String>();
+ ldapProperties.put("hive.server2.authentication.ldap.userDNPattern", "uid=%s,ou=People,dc=example,dc=com");
+ ldapProperties.put("hive.server2.authentication.ldap.userFilter", "user2");
+ initLdapAtn(ldapProperties);
+
+ user = "uid=user1,ou=People,dc=example,dc=com";
+ try {
+ ldapProvider.Authenticate(user, "user1");
+ Assert.fail("testUserFilterNegative: Authentication succeeded for " + user + ",user is expected to fail userfilter");
+ } catch (AuthenticationException e) {
+ assertTrue("testUserFilterNegative: Authentication failed for " + user + " as expected", true);
+ }
+
+ user = "user1";
+ try {
+ ldapProvider.Authenticate(user, "user1");
+ Assert.fail("testUserFilterNegative: Authentication succeeded for " + user + ",user is expected to fail userfilter");
+ } catch (AuthenticationException e) {
+ assertTrue("testUserFilterNegative: Authentication failed for " + user + " as expected", true);
+ }
+
+ ldapProperties = new HashMap<String, String>();
+ ldapProperties.put("hive.server2.authentication.ldap.userDNPattern", "uid=%s,ou=People,dc=example,dc=com");
+ ldapProperties.put("hive.server2.authentication.ldap.userFilter", "user1");
+ initLdapAtn(ldapProperties);
+
+ user = "uid=user2,ou=People,dc=example,dc=com";
+ try {
+ ldapProvider.Authenticate(user, "user2");
+ Assert.fail("testUserFilterNegative: Authentication succeeded for " + user + ",user is expected to fail userfilter");
+ } catch (AuthenticationException e) {
+ assertTrue("testUserFilterNegative: Authentication failed for " + user + " as expected", true);
+ }
+
+ user = "user2";
+ try {
+ ldapProvider.Authenticate(user, "user2");
+ Assert.fail("testUserFilterNegative: Authentication succeeded for " + user + ",user is expected to fail userfilter");
+ } catch (AuthenticationException e) {
+ assertTrue("testUserFilterNegative: Authentication failed for " + user + " as expected", true);
+ }
+
+ ldapProperties = new HashMap<String, String>();
+ ldapProperties.put("hive.server2.authentication.ldap.userDNPattern", "uid=%s,ou=People,dc=example,dc=com");
+ ldapProperties.put("hive.server2.authentication.ldap.userFilter", "user3");
+ initLdapAtn(ldapProperties);
+
+ user = "user1";
+ try {
+ ldapProvider.Authenticate(user, "user1");
+ Assert.fail("testUserFilterNegative: Authentication succeeded for " + user + ",user expected to fail userfilter");
+ } catch (AuthenticationException e) {
+ assertTrue("testUserFilterNegative: Authentication failed for " + user + " as expected", true);
+ }
+
+ user = "uid=user2,ou=People,dc=example,dc=com";
+ try {
+ ldapProvider.Authenticate(user, "user2");
+ Assert.fail("testUserFilterNegative: Authentication succeeded for " + user + ",user expected to fail userfilter");
+ } catch (AuthenticationException e) {
+ assertTrue("testUserFilterNegative: Authentication failed for " + user + " as expected", true);
+ }
+ }
+
+ @Test
+ public void testGroupFilterPositive() throws Exception {
+ String user;
+ Map<String, String> ldapProperties = new HashMap<String, String>();
+ ldapProperties.put("hive.server2.authentication.ldap.userDNPattern", "uid=%s,ou=People,dc=example,dc=com");
+ ldapProperties.put("hive.server2.authentication.ldap.groupDNPattern", "uid=%s,ou=Groups,dc=example,dc=com");
+ ldapProperties.put("hive.server2.authentication.ldap.groupFilter", "group1,group2");
+ initLdapAtn(ldapProperties);
+
+ user = "uid=user1,ou=People,dc=example,dc=com";
+ try {
+ ldapProvider.Authenticate(user, "user1");
+ assertTrue("testGroupFilterPositive: Authentication succeeded for " + user + " as expected", true);
+
+ user = "user1";
+ ldapProvider.Authenticate(user, "user1");
+ assertTrue("testGroupFilterPositive: Authentication succeeded for " + user + " as expected", true);
+
+ user = "uid=user2,ou=People,dc=example,dc=com";
+ ldapProvider.Authenticate(user, "user2");
+ assertTrue("testGroupFilterPositive: Authentication succeeded for " + user + " as expected", true);
+ } catch (AuthenticationException e) {
+ Assert.fail("testGroupFilterPositive: Authentication failed for " + user + ",user expected to pass groupfilter");
+ }
+
+ ldapProperties = new HashMap<String, String>();
+ ldapProperties.put("hive.server2.authentication.ldap.userDNPattern", "uid=%s,ou=People,dc=example,dc=com");
+ ldapProperties.put("hive.server2.authentication.ldap.groupDNPattern", "uid=%s,ou=Groups,dc=example,dc=com");
+ ldapProperties.put("hive.server2.authentication.ldap.groupFilter", "group2");
+ initLdapAtn(ldapProperties);
+
+ user = "uid=user2,ou=People,dc=example,dc=com";
+ try {
+ ldapProvider.Authenticate(user, "user2");
+ assertTrue("testGroupFilterPositive: Authentication succeeded for " + user + " as expected", true);
+ } catch (AuthenticationException e) {
+ Assert.fail("testGroupFilterPositive: Authentication failed for " + user + ",user expected to pass groupfilter");
+ }
+ }
+
+ @Test
+ public void testGroupFilterNegative() throws Exception {
+ String user;
+ Map<String, String> ldapProperties = new HashMap<String, String>();
+ ldapProperties.put("hive.server2.authentication.ldap.userDNPattern", "uid=%s,ou=People,dc=example,dc=com");
+ ldapProperties.put("hive.server2.authentication.ldap.groupDNPattern", "uid=%s,ou=Groups,dc=example,dc=com");
+ ldapProperties.put("hive.server2.authentication.ldap.groupFilter", "group1");
+ initLdapAtn(ldapProperties);
+
+ user = "uid=user2,ou=People,dc=example,dc=com";
+ try {
+ ldapProvider.Authenticate(user, "user2");
+ Assert.fail("testGroupFilterNegative: Authentication succeeded for " + user + ",user expected to fail groupfilter");
+ } catch (AuthenticationException e) {
+ assertTrue("testGroupFilterNegative: Authentication failed for " + user + " as expected", true);
+ }
+
+ ldapProperties = new HashMap<String, String>();
+ ldapProperties.put("hive.server2.authentication.ldap.userDNPattern", "uid=%s,ou=People,dc=example,dc=com");
+ ldapProperties.put("hive.server2.authentication.ldap.groupDNPattern", "uid=%s,ou=Groups,dc=example,dc=com");
+ ldapProperties.put("hive.server2.authentication.ldap.groupFilter", "group2");
+ initLdapAtn(ldapProperties);
+
+ user = "uid=user1,ou=People,dc=example,dc=com";
+ try {
+ ldapProvider.Authenticate(user, "user1");
+ Assert.fail("testGroupFilterNegative: Authentication succeeded for " + user + ",user expected to fail groupfilter");
+ } catch (AuthenticationException e) {
+ assertTrue("testGroupFilterNegative: Authentication failed for " + user + " as expected", true);
+ }
+ }
}
[50/50] [abbrv] hive git commit: HIVE-13419: Merge master into llap
branch
Posted by jd...@apache.org.
HIVE-13419: Merge master into llap branch
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a7b0ca73
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a7b0ca73
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a7b0ca73
Branch: refs/heads/llap
Commit: a7b0ca733e416951ab6c36f71dbe512665477535
Parents: 28d1082 4e9f95a
Author: Jason Dere <jd...@hortonworks.com>
Authored: Mon Apr 4 13:37:14 2016 -0700
Committer: Jason Dere <jd...@hortonworks.com>
Committed: Mon Apr 4 13:37:14 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/ant/GenVectorCode.java | 531 +-
.../org/apache/hadoop/hive/cli/CliDriver.java | 3 +
.../apache/hadoop/hive/common/FileUtils.java | 54 +
.../apache/hadoop/hive/common/ServerUtils.java | 11 +
.../hive/common/type/HiveIntervalDayTime.java | 245 -
.../org/apache/hadoop/hive/conf/HiveConf.java | 31 +-
.../org/apache/hive/common/util/DateUtils.java | 19 -
.../hive/contrib/serde2/MultiDelimitSerDe.java | 23 +-
data/files/alltypesorc3xcols | Bin 0 -> 1504592 bytes
data/files/timestamps.txt | 50 +
.../deployers/config/hive/hive-site.mysql.xml | 24 +-
.../hive/hcatalog/streaming/HiveEndPoint.java | 11 +
.../hcatalog/templeton/SecureProxySupport.java | 46 +-
.../antlr4/org/apache/hive/hplsql/Hplsql.g4 | 266 +-
.../main/java/org/apache/hive/hplsql/Conf.java | 2 +-
.../main/java/org/apache/hive/hplsql/Conn.java | 3 +-
.../java/org/apache/hive/hplsql/Converter.java | 15 +-
.../main/java/org/apache/hive/hplsql/Exec.java | 106 +-
.../java/org/apache/hive/hplsql/Expression.java | 13 +
.../main/java/org/apache/hive/hplsql/File.java | 11 +
.../main/java/org/apache/hive/hplsql/Ftp.java | 415 +
.../main/java/org/apache/hive/hplsql/Meta.java | 35 +-
.../java/org/apache/hive/hplsql/Package.java | 3 +
.../main/java/org/apache/hive/hplsql/Row.java | 4 +-
.../java/org/apache/hive/hplsql/Select.java | 6 +-
.../main/java/org/apache/hive/hplsql/Stmt.java | 167 +-
.../main/java/org/apache/hive/hplsql/Utils.java | 40 +
.../main/java/org/apache/hive/hplsql/Var.java | 18 +-
.../apache/hive/hplsql/functions/Function.java | 40 +-
.../hive/hplsql/functions/FunctionDatetime.java | 40 +
.../hive/hplsql/functions/FunctionString.java | 26 +-
.../org/apache/hive/hplsql/TestHplsqlLocal.java | 28 +-
.../apache/hive/hplsql/TestHplsqlOffline.java | 25 +
.../test/queries/db/create_drop_database.sql | 5 +
.../queries/db/create_procedure_no_params.sql | 25 +
hplsql/src/test/queries/db/describe.sql | 3 +
hplsql/src/test/queries/db/execute.sql | 7 +
hplsql/src/test/queries/db/expression.sql | 1 +
hplsql/src/test/queries/db/for.sql | 1 +
hplsql/src/test/queries/db/insert.sql | 3 +
hplsql/src/test/queries/db/insert_directory.sql | 12 +
hplsql/src/test/queries/db/schema.sql | 10 +-
hplsql/src/test/queries/db/truncate_table.sql | 2 +
.../src/test/queries/local/create_function3.sql | 58 +
.../src/test/queries/local/create_function4.sql | 19 +
.../test/queries/local/create_procedure3.sql | 29 +
hplsql/src/test/queries/local/declare3.sql | 7 +
hplsql/src/test/queries/local/if.sql | 6 +-
hplsql/src/test/queries/local/interval.sql | 4 +-
hplsql/src/test/queries/local/replace.sql | 1 +
.../queries/offline/create_table_mssql2.sql | 33 +
.../test/queries/offline/create_table_mysql.sql | 5 +
.../test/queries/offline/create_table_ora2.sql | 6 +
.../test/queries/offline/create_table_pg.sql | 5 +
hplsql/src/test/queries/offline/update.sql | 33 +
.../results/db/create_drop_database.out.txt | 8 +
.../results/db/create_procedure_mssql.out.txt | 2 +-
.../db/create_procedure_no_params.out.txt | 10 +
.../db/create_procedure_return_cursor.out.txt | 4 +-
.../db/create_procedure_return_cursor2.out.txt | 4 +-
hplsql/src/test/results/db/describe.out.txt | 12 +
hplsql/src/test/results/db/execute.out.txt | 14 +
hplsql/src/test/results/db/expression.out.txt | 5 +
hplsql/src/test/results/db/for.out.txt | 44 +
hplsql/src/test/results/db/insert.out.txt | 4 +
.../test/results/db/insert_directory.out.txt | 9 +
.../test/results/db/rowtype_attribute.out.txt | 2 +-
.../src/test/results/db/truncate_table.out.txt | 4 +
.../test/results/local/create_function3.out.txt | 22 +
.../test/results/local/create_function4.out.txt | 9 +
.../test/results/local/create_package.out.txt | 2 +-
.../results/local/create_procedure3.out.txt | 31 +
.../local/create_procedure_no_params.out.txt | 12 +-
hplsql/src/test/results/local/declare3.out.txt | 9 +
hplsql/src/test/results/local/if.out.txt | 4 +
hplsql/src/test/results/local/interval.out.txt | 1 +
hplsql/src/test/results/local/replace.out.txt | 1 +
.../results/offline/create_table_mssql2.out.txt | 10 +
.../results/offline/create_table_mysql.out.txt | 4 +
.../results/offline/create_table_ora2.out.txt | 5 +
.../results/offline/create_table_pg.out.txt | 5 +
hplsql/src/test/results/offline/update.out.txt | 34 +
.../listener/TestDbNotificationListener.java | 18 +
.../org/apache/hive/jdbc/miniHS2/MiniHS2.java | 56 +-
.../hive/metastore/TestHiveMetaStore.java | 20 +-
.../jdbc/TestJdbcWithLocalClusterSpark.java | 2 +-
.../apache/hive/jdbc/TestJdbcWithMiniMr.java | 2 +-
...stMultiSessionsHS2WithLocalClusterSpark.java | 6 +-
.../jdbc/TestServiceDiscoveryWithMiniHS2.java | 132 +
.../jdbc/authorization/TestHS2AuthzContext.java | 4 +-
.../authorization/TestJdbcMetadataApiAuth.java | 2 +-
.../TestJdbcWithSQLAuthorization.java | 2 +-
.../cli/session/TestHiveSessionImpl.java | 5 +-
.../test/resources/testconfiguration.properties | 8 +
...SQLStdHiveAuthorizationValidatorForTest.java | 29 +
.../hive/jdbc/ZooKeeperHiveClientHelper.java | 21 +-
.../hadoop/hive/llap/io/api/LlapProxy.java | 2 +
.../impl/LlapZookeeperRegistryImpl.java | 2 +
.../org/apache/hadoop/hive/llap/LlapUtil.java | 26 +
.../apache/hadoop/hive/llap/tez/Converters.java | 1 +
.../hadoop/hive/llap/tez/TestConverters.java | 190 +
llap-server/bin/runLlapDaemon.sh | 5 +-
.../llap/IncrementalObjectSizeEstimator.java | 54 +-
.../hadoop/hive/llap/cache/LlapDataBuffer.java | 12 +-
.../hive/llap/cache/LowLevelCacheImpl.java | 35 +-
.../llap/cache/LowLevelCacheMemoryManager.java | 6 +-
.../llap/cache/LowLevelFifoCachePolicy.java | 4 +-
.../llap/cache/LowLevelLrfuCachePolicy.java | 14 +-
.../hive/llap/cache/SimpleBufferManager.java | 8 +-
.../hive/llap/cli/LlapOptionsProcessor.java | 1 -
.../hadoop/hive/llap/cli/LlapServiceDriver.java | 71 +
.../hive/llap/daemon/impl/LlapDaemon.java | 24 +-
.../impl/StaticPermanentFunctionChecker.java | 70 +
.../hive/llap/io/api/impl/LlapInputFormat.java | 32 +-
.../hive/llap/io/api/impl/LlapIoImpl.java | 21 +-
.../llap/io/decode/OrcColumnVectorProducer.java | 4 +-
.../llap/io/encoded/OrcEncodedDataReader.java | 95 +-
.../hadoop/hive/llap/old/BufferInProgress.java | 82 -
.../apache/hadoop/hive/llap/old/BufferPool.java | 225 -
.../hadoop/hive/llap/old/CachePolicy.java | 34 -
.../apache/hadoop/hive/llap/old/ChunkPool.java | 237 -
.../resources/llap-daemon-log4j2.properties | 14 +-
.../hive/metastore/MetaStoreDirectSql.java | 53 +-
.../hadoop/hive/metastore/ObjectStore.java | 10 +-
.../hive/metastore/StatObjectConverter.java | 2 +-
.../hadoop/hive/metastore/hbase/HBaseStore.java | 4 +-
.../hadoop/hive/metastore/hbase/HBaseUtils.java | 8 +-
.../hadoop/hive/metastore/hbase/StatsCache.java | 20 +-
.../stats/BinaryColumnStatsAggregator.java | 43 +-
.../stats/BooleanColumnStatsAggregator.java | 42 +-
.../hbase/stats/ColumnStatsAggregator.java | 12 +-
.../stats/ColumnStatsAggregatorFactory.java | 8 +-
.../stats/DecimalColumnStatsAggregator.java | 340 +-
.../stats/DoubleColumnStatsAggregator.java | 307 +-
.../hbase/stats/IExtrapolatePartStatus.java | 30 +
.../hbase/stats/LongColumnStatsAggregator.java | 305 +-
.../stats/StringColumnStatsAggregator.java | 85 +-
.../hive/metastore/txn/CompactionInfo.java | 4 +
.../metastore/txn/CompactionTxnHandler.java | 127 +-
.../hadoop/hive/metastore/txn/TxnDbUtil.java | 18 +-
.../hadoop/hive/metastore/txn/TxnHandler.java | 366 +-
.../hadoop/hive/metastore/txn/TxnStore.java | 33 +-
.../hadoop/hive/metastore/txn/TxnUtils.java | 99 +-
.../metastore/txn/ValidCompactorTxnList.java | 2 +-
...stHBaseAggregateStatsCacheWithBitVector.java | 6 +-
.../TestHBaseAggregateStatsExtrapolation.java | 717 ++
.../TestHBaseAggregateStatsNDVUniformDist.java | 581 ++
.../hive/metastore/txn/TestTxnHandler.java | 96 +-
.../hadoop/hive/metastore/txn/TestTxnUtils.java | 135 +
odbc/Makefile | 193 -
odbc/pom.xml | 142 -
odbc/src/cpp/HiveColumnDesc.cpp | 190 -
odbc/src/cpp/HiveColumnDesc.h | 73 -
odbc/src/cpp/HiveConnection.h | 58 -
odbc/src/cpp/HiveResultSet.cpp | 616 --
odbc/src/cpp/HiveResultSet.h | 190 -
odbc/src/cpp/HiveRowSet.cpp | 465 --
odbc/src/cpp/HiveRowSet.h | 168 -
odbc/src/cpp/hiveclient.cpp | 294 -
odbc/src/cpp/hiveclient.h | 598 --
odbc/src/cpp/hiveclienthelper.cpp | 86 -
odbc/src/cpp/hiveclienthelper.h | 132 -
odbc/src/cpp/hiveconstants.h | 83 -
odbc/src/cpp/thriftserverconstants.h | 64 -
odbc/src/test/hiveclienttest.c | 1395 ----
odbc/testdata/dataset1.input | 2 -
odbc/testdata/dataset2.input | 1 -
odbc/testdata/dataset_types.input | 2 -
orc/src/java/org/apache/orc/OrcFile.java | 21 +-
.../java/org/apache/orc/impl/WriterImpl.java | 42 +-
packaging/src/main/assembly/src.xml | 1 -
pom.xml | 1 -
...eColumnArithmeticIntervalYearMonthColumn.txt | 56 +-
...eColumnArithmeticIntervalYearMonthScalar.txt | 55 +-
.../DateColumnArithmeticTimestampColumn.txt | 141 +-
.../DateColumnArithmeticTimestampColumnBase.txt | 171 -
.../DateColumnArithmeticTimestampScalar.txt | 113 +-
.../DateColumnArithmeticTimestampScalarBase.txt | 137 -
...eScalarArithmeticIntervalYearMonthColumn.txt | 53 +-
.../DateScalarArithmeticTimestampColumn.txt | 108 +-
.../DateScalarArithmeticTimestampColumnBase.txt | 147 -
...ayTimeColumnCompareIntervalDayTimeColumn.txt | 52 -
...ayTimeColumnCompareIntervalDayTimeScalar.txt | 55 -
...ayTimeScalarCompareIntervalDayTimeColumn.txt | 55 -
...erLongDoubleColumnCompareTimestampColumn.txt | 2 +-
...erLongDoubleColumnCompareTimestampScalar.txt | 4 +-
...erLongDoubleScalarCompareTimestampColumn.txt | 4 +
.../FilterTimestampColumnBetween.txt | 11 +-
...terTimestampColumnCompareTimestampColumn.txt | 417 +-
...imestampColumnCompareTimestampColumnBase.txt | 429 -
...terTimestampColumnCompareTimestampScalar.txt | 128 +-
...imestampColumnCompareTimestampScalarBase.txt | 145 -
...erTimestampScalarCompareLongDoubleColumn.txt | 3 +-
...terTimestampScalarCompareTimestampColumn.txt | 132 +-
...imestampScalarCompareTimestampColumnBase.txt | 147 -
...ayTimeColumnCompareIntervalDayTimeColumn.txt | 54 -
...ayTimeColumnCompareIntervalDayTimeScalar.txt | 57 -
...ayTimeScalarCompareIntervalDayTimeColumn.txt | 57 -
...ervalYearMonthColumnArithmeticDateColumn.txt | 55 +-
...ervalYearMonthColumnArithmeticDateScalar.txt | 51 +-
...YearMonthColumnArithmeticTimestampColumn.txt | 63 +-
...YearMonthColumnArithmeticTimestampScalar.txt | 48 +-
...ervalYearMonthScalarArithmeticDateColumn.txt | 51 +-
...YearMonthScalarArithmeticTimestampColumn.txt | 55 +-
.../LongDoubleColumnCompareTimestampColumn.txt | 1 -
.../LongDoubleColumnCompareTimestampScalar.txt | 3 +-
.../LongDoubleScalarCompareTimestampColumn.txt | 1 +
.../TimestampColumnArithmeticDateColumn.txt | 138 +-
.../TimestampColumnArithmeticDateColumnBase.txt | 172 -
.../TimestampColumnArithmeticDateScalar.txt | 98 +-
.../TimestampColumnArithmeticDateScalarBase.txt | 126 -
...pColumnArithmeticIntervalYearMonthColumn.txt | 59 +-
...pColumnArithmeticIntervalYearMonthScalar.txt | 41 +-
...TimestampColumnArithmeticTimestampColumn.txt | 128 +-
...stampColumnArithmeticTimestampColumnBase.txt | 152 -
...TimestampColumnArithmeticTimestampScalar.txt | 96 +-
...stampColumnArithmeticTimestampScalarBase.txt | 125 -
.../TimestampColumnCompareLongDoubleScalar.txt | 1 +
.../TimestampColumnCompareTimestampColumn.txt | 122 +-
...imestampColumnCompareTimestampColumnBase.txt | 140 -
.../TimestampColumnCompareTimestampScalar.txt | 114 +-
...imestampColumnCompareTimestampScalarBase.txt | 131 -
.../TimestampScalarArithmeticDateColumn.txt | 117 +-
.../TimestampScalarArithmeticDateColumnBase.txt | 151 -
...pScalarArithmeticIntervalYearMonthColumn.txt | 62 +-
...TimestampScalarArithmeticTimestampColumn.txt | 103 +-
...stampScalarArithmeticTimestampColumnBase.txt | 139 -
.../TimestampScalarCompareLongDoubleColumn.txt | 4 +-
.../TimestampScalarCompareTimestampColumn.txt | 115 +-
...imestampScalarCompareTimestampColumnBase.txt | 132 -
.../VectorUDAFMinMaxIntervalDayTime.txt | 454 +
.../UDAFTemplates/VectorUDAFMinMaxTimestamp.txt | 31 +-
.../org/apache/hadoop/hive/llap/DebugUtils.java | 43 -
.../org/apache/hadoop/hive/llap/LogLevels.java | 53 -
.../java/org/apache/hadoop/hive/ql/Driver.java | 26 +-
.../org/apache/hadoop/hive/ql/QueryDisplay.java | 11 +-
.../org/apache/hadoop/hive/ql/exec/DDLTask.java | 16 +-
.../hadoop/hive/ql/exec/FunctionRegistry.java | 7 +
.../hadoop/hive/ql/exec/OperatorUtils.java | 45 +-
.../hive/ql/exec/OrcFileMergeOperator.java | 4 +-
.../ql/exec/SparkHashTableSinkOperator.java | 8 +-
.../hadoop/hive/ql/exec/TableScanOperator.java | 19 +-
.../persistence/HybridHashTableContainer.java | 40 +-
.../ql/exec/persistence/KeyValueContainer.java | 25 +-
.../ql/exec/persistence/ObjectContainer.java | 24 +-
.../hive/ql/exec/persistence/RowContainer.java | 34 +-
.../ql/exec/spark/HiveSparkClientFactory.java | 6 +-
.../ql/exec/spark/SparkReduceRecordHandler.java | 2 +
.../hadoop/hive/ql/exec/tez/DagUtils.java | 1 +
.../hadoop/hive/ql/exec/tez/TezJobMonitor.java | 2 +-
.../hive/ql/exec/vector/TimestampUtils.java | 8 +
.../hive/ql/exec/vector/VectorAssignRow.java | 23 +-
.../exec/vector/VectorColumnAssignFactory.java | 19 +-
.../ql/exec/vector/VectorColumnSetInfo.java | 23 +-
.../hive/ql/exec/vector/VectorCopyRow.java | 32 +
.../ql/exec/vector/VectorDeserializeRow.java | 13 +-
.../exec/vector/VectorExpressionDescriptor.java | 6 +-
.../hive/ql/exec/vector/VectorExtractRow.java | 24 +-
.../ql/exec/vector/VectorGroupByOperator.java | 2 +-
.../ql/exec/vector/VectorGroupKeyHelper.java | 12 +
.../ql/exec/vector/VectorHashKeyWrapper.java | 82 +-
.../exec/vector/VectorHashKeyWrapperBatch.java | 112 +-
.../hive/ql/exec/vector/VectorSerializeRow.java | 7 +-
.../ql/exec/vector/VectorizationContext.java | 27 +-
.../ql/exec/vector/VectorizedBatchUtil.java | 20 +-
.../ql/exec/vector/VectorizedRowBatchCtx.java | 12 +-
.../expressions/CastDecimalToTimestamp.java | 3 +-
.../expressions/CastDoubleToTimestamp.java | 17 +-
.../vector/expressions/CastLongToTimestamp.java | 8 +-
.../CastMillisecondsLongToTimestamp.java | 22 +-
.../CastStringToIntervalDayTime.java | 8 +-
.../expressions/CastTimestampToBoolean.java | 4 +-
.../vector/expressions/CastTimestampToDate.java | 2 +-
.../expressions/CastTimestampToDecimal.java | 9 +-
.../expressions/CastTimestampToDouble.java | 13 +-
.../vector/expressions/CastTimestampToLong.java | 12 +-
.../expressions/ConstantVectorExpression.java | 36 +-
.../expressions/DateColSubtractDateColumn.java | 80 +-
.../expressions/DateColSubtractDateScalar.java | 51 +-
.../DateScalarSubtractDateColumn.java | 52 +-
.../FilterTimestampColumnInList.java | 27 +-
.../IfExprIntervalDayTimeColumnColumn.java | 103 +-
.../IfExprIntervalDayTimeColumnScalar.java | 94 +-
.../IfExprIntervalDayTimeScalarColumn.java | 96 +-
.../IfExprIntervalDayTimeScalarScalar.java | 84 +-
.../IfExprTimestampColumnColumnBase.java | 8 +-
.../IfExprTimestampColumnScalar.java | 3 +-
.../IfExprTimestampColumnScalarBase.java | 14 +-
.../IfExprTimestampScalarColumn.java | 3 +-
.../IfExprTimestampScalarColumnBase.java | 15 +-
.../IfExprTimestampScalarScalar.java | 3 +-
.../IfExprTimestampScalarScalarBase.java | 13 +-
.../expressions/LongColEqualLongScalar.java | 2 +-
.../LongColGreaterEqualLongScalar.java | 2 +-
.../expressions/LongColGreaterLongScalar.java | 2 +-
.../expressions/LongColLessEqualLongScalar.java | 2 +-
.../expressions/LongColLessLongScalar.java | 2 +-
.../expressions/LongColNotEqualLongScalar.java | 2 +-
.../expressions/LongScalarEqualLongColumn.java | 2 +-
.../LongScalarGreaterEqualLongColumn.java | 2 +-
.../LongScalarGreaterLongColumn.java | 2 +-
.../LongScalarLessEqualLongColumn.java | 2 +-
.../expressions/LongScalarLessLongColumn.java | 2 +-
.../LongScalarNotEqualLongColumn.java | 2 +-
.../ql/exec/vector/expressions/NullUtil.java | 26 +
.../expressions/TimestampColumnInList.java | 29 +-
.../expressions/VectorExpressionWriter.java | 6 +-
.../VectorExpressionWriterFactory.java | 124 +-
.../expressions/VectorUDFDateAddColCol.java | 2 +-
.../expressions/VectorUDFDateAddColScalar.java | 2 +-
.../expressions/VectorUDFDateDiffColCol.java | 10 +-
.../expressions/VectorUDFDateDiffColScalar.java | 2 +-
.../expressions/VectorUDFDateDiffScalarCol.java | 2 +-
.../expressions/VectorUDFDateTimestamp.java | 2 +-
.../expressions/VectorUDFUnixTimeStampDate.java | 7 +-
.../VectorUDFUnixTimeStampTimestamp.java | 5 +-
.../aggregates/VectorUDAFAvgTimestamp.java | 40 +-
.../aggregates/VectorUDAFStdPopTimestamp.java | 24 +-
.../aggregates/VectorUDAFStdSampTimestamp.java | 27 +-
.../aggregates/VectorUDAFVarPopTimestamp.java | 24 +-
.../aggregates/VectorUDAFVarSampTimestamp.java | 24 +-
.../mapjoin/VectorMapJoinCommonOperator.java | 2 +-
.../mapjoin/VectorMapJoinRowBytesContainer.java | 24 +-
.../ql/exec/vector/udf/VectorUDFAdaptor.java | 13 +-
.../hadoop/hive/ql/io/HiveInputFormat.java | 15 +-
.../hadoop/hive/ql/io/orc/OrcInputFormat.java | 14 +-
.../hadoop/hive/ql/io/orc/RecordReaderImpl.java | 15 +-
.../hive/ql/io/orc/encoded/EncodedReader.java | 2 +-
.../ql/io/orc/encoded/EncodedReaderImpl.java | 131 +-
.../io/parquet/convert/HiveSchemaConverter.java | 10 +-
.../parquet/read/DataWritableReadSupport.java | 75 +-
.../ql/io/parquet/serde/ParquetHiveSerDe.java | 11 +-
.../apache/hadoop/hive/ql/metadata/Hive.java | 252 +-
.../formatting/MetaDataFormatUtils.java | 8 +-
.../hadoop/hive/ql/optimizer/Optimizer.java | 4 +-
.../ql/optimizer/ReduceSinkMapJoinProc.java | 24 +-
.../hive/ql/optimizer/StatsOptimizer.java | 14 +-
.../ql/optimizer/calcite/HiveRelFactories.java | 5 -
.../calcite/reloperators/HiveAggregate.java | 9 +-
.../rules/HivePointLookupOptimizerRule.java | 381 +
.../ql/optimizer/pcr/PcrExprProcFactory.java | 103 +-
.../hive/ql/optimizer/physical/LlapDecider.java | 2 +-
.../hive/ql/optimizer/physical/Vectorizer.java | 7 +
.../stats/annotation/StatsRulesProcFactory.java | 3 +-
.../hadoop/hive/ql/parse/CalcitePlanner.java | 71 +-
.../hive/ql/parse/DDLSemanticAnalyzer.java | 15 +-
.../hadoop/hive/ql/parse/GenTezUtils.java | 3 +-
.../hadoop/hive/ql/parse/ParseContext.java | 5 +
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 216 +-
.../apache/hadoop/hive/ql/parse/TableMask.java | 127 +
.../org/apache/hadoop/hive/ql/plan/MapWork.java | 4 +-
.../plugin/HiveAuthorizationValidator.java | 11 +
.../authorization/plugin/HiveAuthorizer.java | 69 +
.../plugin/HiveAuthorizerImpl.java | 23 +
.../authorization/plugin/HiveV1Authorizer.java | 25 +
.../sqlstd/DummyHiveAuthorizationValidator.java | 23 +
.../SQLStdHiveAuthorizationValidator.java | 22 +
.../hadoop/hive/ql/session/SessionState.java | 2 +-
.../ql/txn/AcidCompactionHistoryService.java | 7 +
.../hive/ql/txn/AcidHouseKeeperService.java | 7 +
.../hadoop/hive/ql/txn/compactor/Cleaner.java | 68 +-
.../hive/ql/txn/compactor/CompactorThread.java | 5 +
.../hadoop/hive/ql/txn/compactor/Initiator.java | 28 +-
.../hadoop/hive/ql/txn/compactor/Worker.java | 9 +-
.../hadoop/hive/ql/util/DateTimeMath.java | 214 +-
.../apache/hadoop/hive/ql/TestTxnCommands2.java | 54 +-
.../ql/exec/persistence/TestHashPartition.java | 3 +-
.../TestTimestampWritableAndColumnVector.java | 68 +
.../TestVectorExpressionWriters.java | 23 +-
.../TestVectorFilterExpressions.java | 1 -
.../expressions/TestVectorMathFunctions.java | 53 +-
.../TestVectorTimestampExpressions.java | 26 +-
.../vector/expressions/TestVectorTypeCasts.java | 216 +-
.../TestVectorMapJoinRowBytesContainer.java | 3 +-
.../FakeVectorRowBatchFromObjectIterables.java | 3 +-
.../vector/util/VectorizedRowGroupGenUtil.java | 14 +-
.../hive/ql/io/orc/TestColumnStatistics.java | 5 +-
.../hive/ql/io/orc/TestInputOutputFormat.java | 97 +-
.../hadoop/hive/ql/io/orc/TestOrcFile.java | 5 +-
.../hive/ql/io/orc/TestVectorOrcFile.java | 22 +-
.../hive/ql/lockmgr/TestDbTxnManager.java | 6 +
.../hive/ql/lockmgr/TestDbTxnManager2.java | 28 +
.../ql/optimizer/physical/TestVectorizer.java | 5 +
.../distinct_windowing_failure1.q | 20 +
.../distinct_windowing_failure2.q | 22 +
.../queries/clientnegative/right_side_join.q | 12 +
.../clientpositive/auto_join19_inclause.q | 18 +
.../clientpositive/bucket_map_join_tez1.q | 27 +
...umn_names_with_leading_and_trailing_spaces.q | 13 +
.../queries/clientpositive/distinct_windowing.q | 39 +
.../queries/clientpositive/filter_in_or_dup.q | 19 +
ql/src/test/queries/clientpositive/llap_udf.q | 6 +-
ql/src/test/queries/clientpositive/masking_1.q | 27 +
ql/src/test/queries/clientpositive/masking_2.q | 17 +
ql/src/test/queries/clientpositive/masking_3.q | 27 +
ql/src/test/queries/clientpositive/masking_4.q | 30 +
ql/src/test/queries/clientpositive/masking_5.q | 22 +
.../clientpositive/masking_disablecbo_1.q | 28 +
.../clientpositive/masking_disablecbo_2.q | 18 +
.../clientpositive/masking_disablecbo_3.q | 28 +
.../clientpositive/masking_disablecbo_4.q | 31 +
.../clientpositive/multi_insert_with_join.q | 29 +
.../test/queries/clientpositive/orc_merge12.q | 51 +
.../clientpositive/parquet_schema_evolution.q | 14 +
.../sample_islocalmode_hook_use_metadata.q | 48 +
.../queries/clientpositive/schema_evol_stats.q | 50 +
.../queries/clientpositive/vector_complex_all.q | 43 +
.../clientpositive/vector_complex_join.q | 29 +
.../clientpositive/vector_interval_arithmetic.q | 174 +
.../vector_number_compare_projection.q | 192 +
.../clientpositive/vector_tablesample_rows.q | 38 +
.../clientnegative/dbtxnmgr_nodblock.q.out | 2 +
.../clientnegative/dbtxnmgr_nodbunlock.q.out | 2 +
.../distinct_windowing_failure1.q.out | 47 +
.../distinct_windowing_failure2.q.out | 47 +
.../lockneg_query_tbl_in_locked_db.q.out | 6 +
.../lockneg_try_db_lock_conflict.q.out | 6 +
.../lockneg_try_drop_locked_db.q.out | 4 +
.../lockneg_try_lock_db_in_use.q.out | 6 +
.../clientnegative/right_side_join.q.out | 1 +
.../clientpositive/auto_join19_inclause.q.out | 130 +
...names_with_leading_and_trailing_spaces.q.out | 80 +
.../clientpositive/constprog_semijoin.q.out | 4 +-
.../clientpositive/dbtxnmgr_showlocks.q.out | 6 +-
.../clientpositive/distinct_windowing.q.out | 451 +
.../dynpart_sort_optimization_acid.q.out | 4 +-
.../clientpositive/filter_in_or_dup.q.out | 96 +
.../llap/bucket_map_join_tez1.q.out | 343 +
.../llap/bucket_map_join_tez2.q.out | 15 +
.../results/clientpositive/llap/cte_3.q.out | 4 +
.../results/clientpositive/llap/cte_5.q.out | 2 +
.../results/clientpositive/llap/cte_mat_1.q.out | 2 +
.../results/clientpositive/llap/cte_mat_2.q.out | 2 +
.../results/clientpositive/llap/cte_mat_3.q.out | 3 +
.../results/clientpositive/llap/cte_mat_4.q.out | 6 +
.../results/clientpositive/llap/cte_mat_5.q.out | 3 +
.../llap/dynamic_partition_pruning.q.out | 95 +
.../llap/dynamic_partition_pruning_2.q.out | 16 +
.../llap/hybridgrace_hashjoin_1.q.out | 24 +
.../llap/hybridgrace_hashjoin_2.q.out | 36 +
.../results/clientpositive/llap/llap_udf.q.out | 3 +
.../clientpositive/llap/llapdecider.q.out | 12 +
.../clientpositive/llap/lvj_mapjoin.q.out | 2 +
.../clientpositive/llap/mapjoin_decimal.q.out | 2 +
.../test/results/clientpositive/llap/mrr.q.out | 10 +
.../llap/tez_bmj_schema_evolution.q.out | 2 +
.../results/clientpositive/llap/tez_dml.q.out | 3 +
.../llap/tez_dynpart_hashjoin_1.q.out | 12 +
.../llap/tez_dynpart_hashjoin_2.q.out | 9 +
.../results/clientpositive/llap/tez_join.q.out | 2 +
.../clientpositive/llap/tez_join_hash.q.out | 6 +
.../llap/tez_join_result_complex.q.out | 4 +
.../clientpositive/llap/tez_join_tests.q.out | 3 +
.../clientpositive/llap/tez_joins_explain.q.out | 3 +
.../clientpositive/llap/tez_self_join.q.out | 3 +
.../results/clientpositive/llap/tez_smb_1.q.out | 6 +
.../clientpositive/llap/tez_smb_main.q.out | 24 +
.../results/clientpositive/llap/tez_union.q.out | 28 +
.../clientpositive/llap/tez_union2.q.out | 3 +
.../llap/tez_union_dynamic_partition.q.out | 2 +
.../llap/tez_union_group_by.q.out | 5 +
.../llap/tez_union_multiinsert.q.out | 13 +
.../llap/tez_vector_dynpart_hashjoin_1.q.out | 12 +
.../llap/tez_vector_dynpart_hashjoin_2.q.out | 9 +
.../llap/vector_join_part_col_char.q.out | 2 +
.../vectorized_dynamic_partition_pruning.q.out | 89 +
.../test/results/clientpositive/masking_1.q.out | 466 ++
.../test/results/clientpositive/masking_2.q.out | 321 +
.../test/results/clientpositive/masking_3.q.out | 7765 ++++++++++++++++++
.../test/results/clientpositive/masking_4.q.out | 233 +
.../test/results/clientpositive/masking_5.q.out | 189 +
.../clientpositive/masking_disablecbo_1.q.out | 462 ++
.../clientpositive/masking_disablecbo_2.q.out | 355 +
.../clientpositive/masking_disablecbo_3.q.out | 7737 +++++++++++++++++
.../clientpositive/masking_disablecbo_4.q.out | 229 +
.../clientpositive/multi_insert_with_join.q.out | 128 +
.../results/clientpositive/orc_merge12.q.out | 606 ++
.../parquet_map_null.q.java1.8.out | 1 +
.../parquet_schema_evolution.q.out | 65 +
.../clientpositive/parquet_type_promotion.q.out | 2 +-
.../results/clientpositive/perf/query13.q.out | 14 +-
.../results/clientpositive/perf/query27.q.out | 2 +-
.../results/clientpositive/perf/query34.q.out | 2 +-
.../results/clientpositive/perf/query48.q.out | 14 +-
.../results/clientpositive/perf/query68.q.out | 2 +-
.../results/clientpositive/perf/query73.q.out | 2 +-
.../results/clientpositive/perf/query79.q.out | 2 +-
.../results/clientpositive/perf/query82.q.out | 2 +-
.../results/clientpositive/perf/query85.q.out | 26 +-
.../results/clientpositive/pointlookup2.q.out | 38 +-
.../results/clientpositive/pointlookup3.q.out | 50 +-
.../results/clientpositive/pointlookup4.q.out | 2 +-
.../sample_islocalmode_hook_use_metadata.q.out | 230 +
.../clientpositive/schema_evol_stats.q.out | 392 +
.../spark/bucket_map_join_tez1.q.out | 306 +
.../spark/constprog_semijoin.q.out | 4 +-
.../spark/multi_insert_with_join.q.out | 128 +
.../tez/bucket_map_join_tez1.q.out | 294 +
.../clientpositive/tez/bucketpruning1.q.out | 8 +-
...names_with_leading_and_trailing_spaces.q.out | 80 +
.../clientpositive/tez/constprog_semijoin.q.out | 4 +-
.../clientpositive/tez/explainuser_1.q.out | 92 +-
.../clientpositive/tez/llapdecider.q.out | 12 +
.../clientpositive/tez/orc_merge12.q.out | 606 ++
.../clientpositive/tez/schema_evol_stats.q.out | 384 +
.../clientpositive/tez/vector_complex_all.q.out | 254 +
.../tez/vector_complex_join.q.out | 227 +
.../tez/vector_interval_arithmetic.q.out | 1086 +++
.../tez/vector_mr_diff_schema_alias.q.out | 2 +-
.../clientpositive/tez/vectorized_casts.q.out | 18 +-
.../tez/vectorized_timestamp.q.out | 157 +
.../clientpositive/vector_complex_all.q.out | 235 +
.../clientpositive/vector_complex_join.q.out | 225 +
.../vector_interval_arithmetic.q.out | 1027 +++
.../vector_mr_diff_schema_alias.q.out | 2 +-
.../vector_number_compare_projection.q.out | 718 ++
.../vector_tablesample_rows.q.out | 371 +
.../clientpositive/vectorized_casts.q.out | 18 +-
.../hadoop/hive/serde2/avro/AvroSerdeUtils.java | 20 +-
.../hive/serde2/io/TimestampWritable.java | 71 +-
.../auth/LdapAuthenticationProviderImpl.java | 317 +-
.../service/cli/session/HiveSessionImpl.java | 13 +-
.../apache/hive/service/server/HiveServer2.java | 39 +-
.../auth/TestLdapAtnProviderWithMiniDS.java | 200 +-
.../apache/hive/service/cli/CLIServiceTest.java | 8 +
.../org/apache/hadoop/fs/ProxyFileSystem.java | 5 +-
.../common/io/encoded/EncodedColumnBatch.java | 2 +-
.../hive/common/type/HiveIntervalDayTime.java | 253 +
.../hadoop/hive/common/type/PisaTimestamp.java | 609 --
.../hadoop/hive/common/type/RandomTypeUtil.java | 70 +-
.../hive/ql/exec/vector/ColumnVector.java | 2 +-
.../vector/IntervalDayTimeColumnVector.java | 348 +
.../ql/exec/vector/TimestampColumnVector.java | 341 +-
.../hive/common/util/IntervalDayTimeUtils.java | 77 +
.../hive/common/type/TestPisaTimestamp.java | 118 -
.../exec/vector/TestTimestampColumnVector.java | 117 +
.../ptest2/conf/example-apache-trunk.properties | 2 +-
537 files changed, 42116 insertions(+), 12248 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/a7b0ca73/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/a7b0ca73/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/a7b0ca73/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapZookeeperRegistryImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/a7b0ca73/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/a7b0ca73/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
----------------------------------------------------------------------
[31/50] [abbrv] hive git commit: HIVE-13111: Fix timestamp /
interval_day_time wrong results with HIVE-9862 (Matt McCline,
reviewed by Jason Dere)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticTimestampColumnBase.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticTimestampColumnBase.txt b/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticTimestampColumnBase.txt
deleted file mode 100644
index 0ff9226..0000000
--- a/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticTimestampColumnBase.txt
+++ /dev/null
@@ -1,139 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
-
-import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
-import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
-import org.apache.hadoop.hive.ql.exec.vector.*;
-
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
-/*
- * Because of the templatized nature of the code, either or both
- * of these ColumnVector imports may be needed. Listing both of them
- * rather than using ....vectorization.*;
- */
-import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil;
-import org.apache.hadoop.hive.ql.util.DateTimeMath;
-
-/**
- * Generated from template TimestampScalarArithmeticTimestampColumnBase.txt.
- * Implements a vectorized arithmetic operator with a scalar on the left and a
- * column vector on the right. The result is output to an output column vector.
- */
-public abstract class <BaseClassName> extends VectorExpression {
-
- private static final long serialVersionUID = 1L;
-
- private int colNum;
- private PisaTimestamp value;
- private int outputColumn;
- private PisaTimestamp scratchPisaTimestamp;
- private DateTimeMath dtm = new DateTimeMath();
-
- public <BaseClassName>(PisaTimestamp value, int colNum, int outputColumn) {
- this.colNum = colNum;
- this.value = value;
- this.outputColumn = outputColumn;
- scratchPisaTimestamp = new PisaTimestamp();
- }
-
- public <BaseClassName>() {
- }
-
- @Override
- /**
- * Method to evaluate scalar-column operation in vectorized fashion.
- *
- * @batch a package of rows with each column stored in a vector
- */
- public void evaluate(VectorizedRowBatch batch) {
-
- if (childExpressions != null) {
- super.evaluateChildren(batch);
- }
-
- // Input #2 is type timestamp/interval_day_time.
- TimestampColumnVector inputColVector2 = (TimestampColumnVector) batch.cols[colNum];
-
- // Output is type timestamp/interval_day_time.
- TimestampColumnVector outputColVector = (TimestampColumnVector) batch.cols[outputColumn];
-
- int[] sel = batch.selected;
- boolean[] inputIsNull = inputColVector2.isNull;
- boolean[] outputIsNull = outputColVector.isNull;
- outputColVector.noNulls = inputColVector2.noNulls;
- outputColVector.isRepeating = inputColVector2.isRepeating;
- int n = batch.size;
-
- // return immediately if batch is empty
- if (n == 0) {
- return;
- }
-
- if (inputColVector2.isRepeating) {
- outputColVector.<OperatorMethod>(
- value, inputColVector2.asScratchPisaTimestamp(0), 0);
-
- // Even if there are no nulls, we always copy over entry 0. Simplifies code.
- outputIsNull[0] = inputIsNull[0];
- } else if (inputColVector2.noNulls) {
- if (batch.selectedInUse) {
- for(int j = 0; j != n; j++) {
- int i = sel[j];
- outputColVector.<OperatorMethod>(
- value, inputColVector2.asScratchPisaTimestamp(i), i);
- }
- } else {
- for(int i = 0; i != n; i++) {
- outputColVector.<OperatorMethod>(
- value, inputColVector2.asScratchPisaTimestamp(i), i);
- }
- }
- } else { /* there are nulls */
- if (batch.selectedInUse) {
- for(int j = 0; j != n; j++) {
- int i = sel[j];
- outputColVector.<OperatorMethod>(
- value, inputColVector2.asScratchPisaTimestamp(i), i);
- outputIsNull[i] = inputIsNull[i];
- }
- } else {
- for(int i = 0; i != n; i++) {
- outputColVector.<OperatorMethod>(
- value, inputColVector2.asScratchPisaTimestamp(i), i);
- }
- System.arraycopy(inputIsNull, 0, outputIsNull, 0, n);
- }
- }
-
- NullUtil.setNullOutputEntriesColScalar(outputColVector, batch.selectedInUse, sel, n);
- }
-
- @Override
- public int getOutputColumn() {
- return outputColumn;
- }
-
- @Override
- public String getOutputType() {
- return "timestamp";
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarCompareLongDoubleColumn.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarCompareLongDoubleColumn.txt b/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarCompareLongDoubleColumn.txt
index 9e855e8..6815b5b 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarCompareLongDoubleColumn.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarCompareLongDoubleColumn.txt
@@ -19,8 +19,8 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
import java.sql.Timestamp;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
+import org.apache.hadoop.hive.ql.exec.vector.*;
import org.apache.hadoop.hive.ql.exec.vector.expressions.*;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
@@ -35,7 +35,7 @@ public class <ClassName> extends <BaseClassName> {
private static final long serialVersionUID = 1L;
public <ClassName>(Timestamp value, int colNum, int outputColumn) {
- super(new PisaTimestamp(value).<GetTimestampLongDoubleMethod>(), colNum, outputColumn);
+ super(TimestampColumnVector.<GetTimestampLongDoubleMethod>(value), colNum, outputColumn);
}
public <ClassName>() {
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarCompareTimestampColumn.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarCompareTimestampColumn.txt b/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarCompareTimestampColumn.txt
index df9f3c9..6506c93 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarCompareTimestampColumn.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarCompareTimestampColumn.txt
@@ -21,26 +21,117 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
import java.sql.Timestamp;
import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
-
+import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
+import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.*;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
-
/**
- * Generated from template TimestampColumnCompareTimestampScalar.txt, which covers comparison
- * expressions between a datetime/interval column and a scalar of the same type. The boolean output
- * is stored in a separate boolean column.
+ * Generated from template ScalarCompareTimestamp.txt, which covers comparison
+ * expressions between a long/double scalar and a column. The boolean output is stored in a
+ * separate boolean column.
*/
-public class <ClassName> extends <BaseClassName> {
+public class <ClassName> extends VectorExpression {
private static final long serialVersionUID = 1L;
- public <ClassName>(Timestamp value, int colNum, int outputColumn) {
- super(new PisaTimestamp(value), colNum, outputColumn);
+ private int colNum;
+ private <HiveOperandType> value;
+ private int outputColumn;
+
+ public <ClassName>(<HiveOperandType> value, int colNum, int outputColumn) {
+ this.colNum = colNum;
+ this.value = value;
+ this.outputColumn = outputColumn;
}
public <ClassName>() {
- super();
+ }
+
+ @Override
+ public void evaluate(VectorizedRowBatch batch) {
+
+ if (childExpressions != null) {
+ super.evaluateChildren(batch);
+ }
+
+ // Input #2 is type <OperandType>.
+ <InputColumnVectorType> inputColVector2 = (<InputColumnVectorType>) batch.cols[colNum];
+
+ LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumn];
+
+ int[] sel = batch.selected;
+ boolean[] nullPos = inputColVector2.isNull;
+ boolean[] outNulls = outputColVector.isNull;
+ int n = batch.size;
+ long[] outputVector = outputColVector.vector;
+
+ // return immediately if batch is empty
+ if (n == 0) {
+ return;
+ }
+
+ outputColVector.isRepeating = false;
+ outputColVector.noNulls = inputColVector2.noNulls;
+ if (inputColVector2.noNulls) {
+ if (inputColVector2.isRepeating) {
+ //All must be selected otherwise size would be zero
+ //Repeating property will not change.
+ outputVector[0] = inputColVector2.compareTo(value, 0) <OperatorSymbol> 0 ? 1 : 0;
+ outputColVector.isRepeating = true;
+ } else if (batch.selectedInUse) {
+ for(int j=0; j != n; j++) {
+ int i = sel[j];
+ outputVector[i] = inputColVector2.compareTo(value, i) <OperatorSymbol> 0 ? 1 : 0;
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ outputVector[i] = inputColVector2.compareTo(value, i) <OperatorSymbol> 0 ? 1 : 0;
+ }
+ }
+ } else {
+ if (inputColVector2.isRepeating) {
+ //All must be selected otherwise size would be zero
+ //Repeating property will not change.
+ if (!nullPos[0]) {
+ outputVector[0] = inputColVector2.compareTo(value, 0) <OperatorSymbol> 0 ? 1 : 0;
+ outNulls[0] = false;
+ } else {
+ outNulls[0] = true;
+ }
+ outputColVector.isRepeating = true;
+ } else if (batch.selectedInUse) {
+ for(int j=0; j != n; j++) {
+ int i = sel[j];
+ if (!nullPos[i]) {
+ outputVector[i] = inputColVector2.compareTo(value, i) <OperatorSymbol> 0 ? 1 : 0;
+ outNulls[i] = false;
+ } else {
+ //comparison with null is null
+ outNulls[i] = true;
+ }
+ }
+ } else {
+ System.arraycopy(nullPos, 0, outNulls, 0, n);
+ for(int i = 0; i != n; i++) {
+ if (!nullPos[i]) {
+ outputVector[i] = inputColVector2.compareTo(value, i) <OperatorSymbol> 0 ? 1 : 0;
+ }
+ }
+ }
+ }
+ }
+
+ @Override
+ public int getOutputColumn() {
+ return outputColumn;
+ }
+
+ @Override
+ public String getOutputType() {
+ return "long";
}
@Override
@@ -50,8 +141,8 @@ public class <ClassName> extends <BaseClassName> {
VectorExpressionDescriptor.Mode.PROJECTION)
.setNumArguments(2)
.setArgumentTypes(
- VectorExpressionDescriptor.ArgumentType.getType("timestamp"),
- VectorExpressionDescriptor.ArgumentType.getType("timestamp"))
+ VectorExpressionDescriptor.ArgumentType.getType("<OperandType>"),
+ VectorExpressionDescriptor.ArgumentType.getType("<OperandType>"))
.setInputExpressionTypes(
VectorExpressionDescriptor.InputExpressionType.SCALAR,
VectorExpressionDescriptor.InputExpressionType.COLUMN).build();
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarCompareTimestampColumnBase.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarCompareTimestampColumnBase.txt b/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarCompareTimestampColumnBase.txt
deleted file mode 100644
index bd345e7..0000000
--- a/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarCompareTimestampColumnBase.txt
+++ /dev/null
@@ -1,132 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
-
-import java.sql.Timestamp;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
-
-import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
-import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
-import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
-
-/**
- * Generated from template ScalarCompareTimestamp.txt, which covers comparison
- * expressions between a long/double scalar and a column. The boolean output is stored in a
- * separate boolean column.
- */
-public abstract class <ClassName> extends VectorExpression {
-
- private static final long serialVersionUID = 1L;
-
- private int colNum;
- private PisaTimestamp value;
- private int outputColumn;
-
- public <ClassName>(PisaTimestamp value, int colNum, int outputColumn) {
- this.colNum = colNum;
- this.value = value;
- this.outputColumn = outputColumn;
- }
-
- public <ClassName>() {
- }
-
- @Override
- public void evaluate(VectorizedRowBatch batch) {
-
- if (childExpressions != null) {
- super.evaluateChildren(batch);
- }
-
- TimestampColumnVector inputColVector2 = (TimestampColumnVector) batch.cols[colNum];
- LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumn];
- int[] sel = batch.selected;
- boolean[] nullPos = inputColVector2.isNull;
- boolean[] outNulls = outputColVector.isNull;
- int n = batch.size;
- long[] outputVector = outputColVector.vector;
-
- // return immediately if batch is empty
- if (n == 0) {
- return;
- }
-
- outputColVector.isRepeating = false;
- outputColVector.noNulls = inputColVector2.noNulls;
- if (inputColVector2.noNulls) {
- if (inputColVector2.isRepeating) {
- //All must be selected otherwise size would be zero
- //Repeating property will not change.
- outputVector[0] = inputColVector2.compareTo(value, 0) <OperatorSymbol> 0 ? 1 : 0;
- outputColVector.isRepeating = true;
- } else if (batch.selectedInUse) {
- for(int j=0; j != n; j++) {
- int i = sel[j];
- outputVector[i] = inputColVector2.compareTo(value, i) <OperatorSymbol> 0 ? 1 : 0;
- }
- } else {
- for(int i = 0; i != n; i++) {
- outputVector[i] = inputColVector2.compareTo(value, i) <OperatorSymbol> 0 ? 1 : 0;
- }
- }
- } else {
- if (inputColVector2.isRepeating) {
- //All must be selected otherwise size would be zero
- //Repeating property will not change.
- if (!nullPos[0]) {
- outputVector[0] = inputColVector2.compareTo(value, 0) <OperatorSymbol> 0 ? 1 : 0;
- outNulls[0] = false;
- } else {
- outNulls[0] = true;
- }
- outputColVector.isRepeating = true;
- } else if (batch.selectedInUse) {
- for(int j=0; j != n; j++) {
- int i = sel[j];
- if (!nullPos[i]) {
- outputVector[i] = inputColVector2.compareTo(value, i) <OperatorSymbol> 0 ? 1 : 0;
- outNulls[i] = false;
- } else {
- //comparison with null is null
- outNulls[i] = true;
- }
- }
- } else {
- System.arraycopy(nullPos, 0, outNulls, 0, n);
- for(int i = 0; i != n; i++) {
- if (!nullPos[i]) {
- outputVector[i] = inputColVector2.compareTo(value, i) <OperatorSymbol> 0 ? 1 : 0;
- }
- }
- }
- }
- }
-
- @Override
- public int getOutputColumn() {
- return outputColumn;
- }
-
- @Override
- public String getOutputType() {
- return "long";
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxIntervalDayTime.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxIntervalDayTime.txt b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxIntervalDayTime.txt
new file mode 100644
index 0000000..3cdf7e2
--- /dev/null
+++ b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxIntervalDayTime.txt
@@ -0,0 +1,454 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen;
+
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriter;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriterFactory;
+import org.apache.hadoop.hive.ql.exec.vector.VectorAggregationBufferRow;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.exec.vector.IntervalDayTimeColumnVector;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.plan.AggregationDesc;
+import org.apache.hadoop.hive.ql.util.JavaDataModel;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
+
+/**
+* <ClassName>. Vectorized implementation for MIN/MAX aggregates.
+*/
+@Description(name = "<DescriptionName>",
+ value = "<DescriptionValue>")
+public class <ClassName> extends VectorAggregateExpression {
+
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * class for storing the current aggregate value.
+ */
+ static private final class Aggregation implements AggregationBuffer {
+
+ private static final long serialVersionUID = 1L;
+
+ transient private final HiveIntervalDayTime value;
+
+ /**
+ * Value is explicitly (re)initialized in reset()
+ */
+ transient private boolean isNull = true;
+
+ public Aggregation() {
+ value = new HiveIntervalDayTime();
+ }
+
+ public void checkValue(IntervalDayTimeColumnVector colVector, int index) {
+ if (isNull) {
+ isNull = false;
+ colVector.intervalDayTimeUpdate(this.value, index);
+ } else if (colVector.compareTo(this.value, index) <OperatorSymbol> 0) {
+ colVector.intervalDayTimeUpdate(this.value, index);
+ }
+ }
+
+ @Override
+ public int getVariableSize() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void reset () {
+ isNull = true;
+ this.value.set(0, 0);
+ }
+ }
+
+ private VectorExpression inputExpression;
+ private transient VectorExpressionWriter resultWriter;
+
+ public <ClassName>(VectorExpression inputExpression) {
+ this();
+ this.inputExpression = inputExpression;
+ }
+
+ public <ClassName>() {
+ super();
+ }
+
+ @Override
+ public void init(AggregationDesc desc) throws HiveException {
+ resultWriter = VectorExpressionWriterFactory.genVectorExpressionWritable(
+ desc.getParameters().get(0));
+ }
+
+ private Aggregation getCurrentAggregationBuffer(
+ VectorAggregationBufferRow[] aggregationBufferSets,
+ int aggregrateIndex,
+ int row) {
+ VectorAggregationBufferRow mySet = aggregationBufferSets[row];
+ Aggregation myagg = (Aggregation) mySet.getAggregationBuffer(aggregrateIndex);
+ return myagg;
+ }
+
+ @Override
+ public void aggregateInputSelection(
+ VectorAggregationBufferRow[] aggregationBufferSets,
+ int aggregrateIndex,
+ VectorizedRowBatch batch) throws HiveException {
+
+ int batchSize = batch.size;
+
+ if (batchSize == 0) {
+ return;
+ }
+
+ inputExpression.evaluate(batch);
+
+ IntervalDayTimeColumnVector inputColVector = (IntervalDayTimeColumnVector)batch.
+ cols[this.inputExpression.getOutputColumn()];
+
+ if (inputColVector.noNulls) {
+ if (inputColVector.isRepeating) {
+ iterateNoNullsRepeatingWithAggregationSelection(
+ aggregationBufferSets, aggregrateIndex,
+ inputColVector, batchSize);
+ } else {
+ if (batch.selectedInUse) {
+ iterateNoNullsSelectionWithAggregationSelection(
+ aggregationBufferSets, aggregrateIndex,
+ inputColVector, batch.selected, batchSize);
+ } else {
+ iterateNoNullsWithAggregationSelection(
+ aggregationBufferSets, aggregrateIndex,
+ inputColVector, batchSize);
+ }
+ }
+ } else {
+ if (inputColVector.isRepeating) {
+ if (batch.selectedInUse) {
+ iterateHasNullsRepeatingSelectionWithAggregationSelection(
+ aggregationBufferSets, aggregrateIndex,
+ inputColVector, batchSize, batch.selected, inputColVector.isNull);
+ } else {
+ iterateHasNullsRepeatingWithAggregationSelection(
+ aggregationBufferSets, aggregrateIndex,
+ inputColVector, batchSize, inputColVector.isNull);
+ }
+ } else {
+ if (batch.selectedInUse) {
+ iterateHasNullsSelectionWithAggregationSelection(
+ aggregationBufferSets, aggregrateIndex,
+ inputColVector, batchSize, batch.selected, inputColVector.isNull);
+ } else {
+ iterateHasNullsWithAggregationSelection(
+ aggregationBufferSets, aggregrateIndex,
+ inputColVector, batchSize, inputColVector.isNull);
+ }
+ }
+ }
+ }
+
+ private void iterateNoNullsRepeatingWithAggregationSelection(
+ VectorAggregationBufferRow[] aggregationBufferSets,
+ int aggregrateIndex,
+ IntervalDayTimeColumnVector inputColVector,
+ int batchSize) {
+
+ for (int i=0; i < batchSize; ++i) {
+ Aggregation myagg = getCurrentAggregationBuffer(
+ aggregationBufferSets,
+ aggregrateIndex,
+ i);
+ // Repeating use index 0.
+ myagg.checkValue(inputColVector, 0);
+ }
+ }
+
+ private void iterateNoNullsSelectionWithAggregationSelection(
+ VectorAggregationBufferRow[] aggregationBufferSets,
+ int aggregrateIndex,
+ IntervalDayTimeColumnVector inputColVector,
+ int[] selection,
+ int batchSize) {
+
+ for (int i=0; i < batchSize; ++i) {
+ Aggregation myagg = getCurrentAggregationBuffer(
+ aggregationBufferSets,
+ aggregrateIndex,
+ i);
+ myagg.checkValue(inputColVector, selection[i]);
+ }
+ }
+
+ private void iterateNoNullsWithAggregationSelection(
+ VectorAggregationBufferRow[] aggregationBufferSets,
+ int aggregrateIndex,
+ IntervalDayTimeColumnVector inputColVector,
+ int batchSize) {
+ for (int i=0; i < batchSize; ++i) {
+ Aggregation myagg = getCurrentAggregationBuffer(
+ aggregationBufferSets,
+ aggregrateIndex,
+ i);
+ myagg.checkValue(inputColVector, i);
+ }
+ }
+
+ private void iterateHasNullsRepeatingSelectionWithAggregationSelection(
+ VectorAggregationBufferRow[] aggregationBufferSets,
+ int aggregrateIndex,
+ IntervalDayTimeColumnVector inputColVector,
+ int batchSize,
+ int[] selection,
+ boolean[] isNull) {
+
+ for (int i=0; i < batchSize; ++i) {
+ if (!isNull[selection[i]]) {
+ Aggregation myagg = getCurrentAggregationBuffer(
+ aggregationBufferSets,
+ aggregrateIndex,
+ i);
+ // Repeating use index 0.
+ myagg.checkValue(inputColVector, 0);
+ }
+ }
+
+ }
+
+ private void iterateHasNullsRepeatingWithAggregationSelection(
+ VectorAggregationBufferRow[] aggregationBufferSets,
+ int aggregrateIndex,
+ IntervalDayTimeColumnVector inputColVector,
+ int batchSize,
+ boolean[] isNull) {
+
+ for (int i=0; i < batchSize; ++i) {
+ if (!isNull[i]) {
+ Aggregation myagg = getCurrentAggregationBuffer(
+ aggregationBufferSets,
+ aggregrateIndex,
+ i);
+ // Repeating use index 0.
+ myagg.checkValue(inputColVector, 0);
+ }
+ }
+ }
+
+ private void iterateHasNullsSelectionWithAggregationSelection(
+ VectorAggregationBufferRow[] aggregationBufferSets,
+ int aggregrateIndex,
+ IntervalDayTimeColumnVector inputColVector,
+ int batchSize,
+ int[] selection,
+ boolean[] isNull) {
+
+ for (int j=0; j < batchSize; ++j) {
+ int i = selection[j];
+ if (!isNull[i]) {
+ Aggregation myagg = getCurrentAggregationBuffer(
+ aggregationBufferSets,
+ aggregrateIndex,
+ j);
+ myagg.checkValue(inputColVector, i);
+ }
+ }
+ }
+
+ private void iterateHasNullsWithAggregationSelection(
+ VectorAggregationBufferRow[] aggregationBufferSets,
+ int aggregrateIndex,
+ IntervalDayTimeColumnVector inputColVector,
+ int batchSize,
+ boolean[] isNull) {
+
+ for (int i=0; i < batchSize; ++i) {
+ if (!isNull[i]) {
+ Aggregation myagg = getCurrentAggregationBuffer(
+ aggregationBufferSets,
+ aggregrateIndex,
+ i);
+ myagg.checkValue(inputColVector, i);
+ }
+ }
+ }
+
+ @Override
+ public void aggregateInput(AggregationBuffer agg, VectorizedRowBatch batch)
+ throws HiveException {
+
+ inputExpression.evaluate(batch);
+
+ IntervalDayTimeColumnVector inputColVector = (IntervalDayTimeColumnVector)batch.
+ cols[this.inputExpression.getOutputColumn()];
+
+ int batchSize = batch.size;
+
+ if (batchSize == 0) {
+ return;
+ }
+
+ Aggregation myagg = (Aggregation)agg;
+
+ if (inputColVector.isRepeating) {
+ if (inputColVector.noNulls &&
+ (myagg.isNull || (inputColVector.compareTo(myagg.value, 0) <OperatorSymbol> 0))) {
+ myagg.isNull = false;
+ inputColVector.intervalDayTimeUpdate(myagg.value, 0);
+ }
+ return;
+ }
+
+ if (!batch.selectedInUse && inputColVector.noNulls) {
+ iterateNoSelectionNoNulls(myagg, inputColVector, batchSize);
+ }
+ else if (!batch.selectedInUse) {
+ iterateNoSelectionHasNulls(myagg, inputColVector,
+ batchSize, inputColVector.isNull);
+ }
+ else if (inputColVector.noNulls){
+ iterateSelectionNoNulls(myagg, inputColVector, batchSize, batch.selected);
+ }
+ else {
+ iterateSelectionHasNulls(myagg, inputColVector,
+ batchSize, inputColVector.isNull, batch.selected);
+ }
+ }
+
+ private void iterateSelectionHasNulls(
+ Aggregation myagg,
+ IntervalDayTimeColumnVector inputColVector,
+ int batchSize,
+ boolean[] isNull,
+ int[] selected) {
+
+ for (int j=0; j< batchSize; ++j) {
+ int i = selected[j];
+ if (!isNull[i]) {
+ if (myagg.isNull) {
+ myagg.isNull = false;
+ inputColVector.intervalDayTimeUpdate(myagg.value, i);
+ }
+ else if (inputColVector.compareTo(myagg.value, i) <OperatorSymbol> 0) {
+ inputColVector.intervalDayTimeUpdate(myagg.value, i);
+ }
+ }
+ }
+ }
+
+ private void iterateSelectionNoNulls(
+ Aggregation myagg,
+ IntervalDayTimeColumnVector inputColVector,
+ int batchSize,
+ int[] selected) {
+
+ if (myagg.isNull) {
+ inputColVector.intervalDayTimeUpdate(myagg.value, selected[0]);
+ myagg.isNull = false;
+ }
+
+ for (int i=0; i< batchSize; ++i) {
+ int sel = selected[i];
+ if (inputColVector.compareTo(myagg.value, sel) <OperatorSymbol> 0) {
+ inputColVector.intervalDayTimeUpdate(myagg.value, sel);
+ }
+ }
+ }
+
+ private void iterateNoSelectionHasNulls(
+ Aggregation myagg,
+ IntervalDayTimeColumnVector inputColVector,
+ int batchSize,
+ boolean[] isNull) {
+
+ for(int i=0;i<batchSize;++i) {
+ if (!isNull[i]) {
+ if (myagg.isNull) {
+ inputColVector.intervalDayTimeUpdate(myagg.value, i);
+ myagg.isNull = false;
+ }
+ else if (inputColVector.compareTo(myagg.value, i) <OperatorSymbol> 0) {
+ inputColVector.intervalDayTimeUpdate(myagg.value, i);
+ }
+ }
+ }
+ }
+
+ private void iterateNoSelectionNoNulls(
+ Aggregation myagg,
+ IntervalDayTimeColumnVector inputColVector,
+ int batchSize) {
+ if (myagg.isNull) {
+ inputColVector.intervalDayTimeUpdate(myagg.value, 0);
+ myagg.isNull = false;
+ }
+
+ for (int i=0;i<batchSize;++i) {
+ if (inputColVector.compareTo(myagg.value, i) <OperatorSymbol> 0) {
+ inputColVector.intervalDayTimeUpdate(myagg.value, i);
+ }
+ }
+ }
+
+ @Override
+ public AggregationBuffer getNewAggregationBuffer() throws HiveException {
+ return new Aggregation();
+ }
+
+ @Override
+ public void reset(AggregationBuffer agg) throws HiveException {
+ Aggregation myAgg = (Aggregation) agg;
+ myAgg.reset();
+ }
+
+ @Override
+ public Object evaluateOutput(
+ AggregationBuffer agg) throws HiveException {
+ Aggregation myagg = (Aggregation) agg;
+ if (myagg.isNull) {
+ return null;
+ }
+ else {
+ return resultWriter.writeValue(myagg.value);
+ }
+ }
+
+ @Override
+ public ObjectInspector getOutputObjectInspector() {
+ return resultWriter.getObjectInspector();
+ }
+
+ @Override
+ public int getAggregationBufferFixedSize() {
+ JavaDataModel model = JavaDataModel.get();
+ return JavaDataModel.alignUp(
+ model.object() +
+ model.primitive2(),
+ model.memoryAlign());
+ }
+
+ public VectorExpression getInputExpression() {
+ return inputExpression;
+ }
+
+ public void setInputExpression(VectorExpression inputExpression) {
+ this.inputExpression = inputExpression;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxTimestamp.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxTimestamp.txt b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxTimestamp.txt
index 3cdf405..7e34965 100644
--- a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxTimestamp.txt
+++ b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxTimestamp.txt
@@ -18,7 +18,8 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
+import java.sql.Timestamp;
+
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression;
@@ -49,7 +50,7 @@ public class <ClassName> extends VectorAggregateExpression {
private static final long serialVersionUID = 1L;
- transient private final PisaTimestamp value;
+ transient private final Timestamp value;
/**
* Value is explicitly (re)initialized in reset()
@@ -57,15 +58,15 @@ public class <ClassName> extends VectorAggregateExpression {
transient private boolean isNull = true;
public Aggregation() {
- value = new PisaTimestamp();
+ value = new Timestamp(0);
}
public void checkValue(TimestampColumnVector colVector, int index) {
if (isNull) {
isNull = false;
- colVector.pisaTimestampUpdate(this.value, index);
+ colVector.timestampUpdate(this.value, index);
} else if (colVector.compareTo(this.value, index) <OperatorSymbol> 0) {
- colVector.pisaTimestampUpdate(this.value, index);
+ colVector.timestampUpdate(this.value, index);
}
}
@@ -77,7 +78,7 @@ public class <ClassName> extends VectorAggregateExpression {
@Override
public void reset () {
isNull = true;
- this.value.reset();
+ this.value.setTime(0);
}
}
@@ -311,7 +312,7 @@ public class <ClassName> extends VectorAggregateExpression {
if (inputColVector.noNulls &&
(myagg.isNull || (inputColVector.compareTo(myagg.value, 0) <OperatorSymbol> 0))) {
myagg.isNull = false;
- inputColVector.pisaTimestampUpdate(myagg.value, 0);
+ inputColVector.timestampUpdate(myagg.value, 0);
}
return;
}
@@ -344,10 +345,10 @@ public class <ClassName> extends VectorAggregateExpression {
if (!isNull[i]) {
if (myagg.isNull) {
myagg.isNull = false;
- inputColVector.pisaTimestampUpdate(myagg.value, i);
+ inputColVector.timestampUpdate(myagg.value, i);
}
else if (inputColVector.compareTo(myagg.value, i) <OperatorSymbol> 0) {
- inputColVector.pisaTimestampUpdate(myagg.value, i);
+ inputColVector.timestampUpdate(myagg.value, i);
}
}
}
@@ -360,14 +361,14 @@ public class <ClassName> extends VectorAggregateExpression {
int[] selected) {
if (myagg.isNull) {
- inputColVector.pisaTimestampUpdate(myagg.value, selected[0]);
+ inputColVector.timestampUpdate(myagg.value, selected[0]);
myagg.isNull = false;
}
for (int i=0; i< batchSize; ++i) {
int sel = selected[i];
if (inputColVector.compareTo(myagg.value, sel) <OperatorSymbol> 0) {
- inputColVector.pisaTimestampUpdate(myagg.value, sel);
+ inputColVector.timestampUpdate(myagg.value, sel);
}
}
}
@@ -381,11 +382,11 @@ public class <ClassName> extends VectorAggregateExpression {
for(int i=0;i<batchSize;++i) {
if (!isNull[i]) {
if (myagg.isNull) {
- inputColVector.pisaTimestampUpdate(myagg.value, i);
+ inputColVector.timestampUpdate(myagg.value, i);
myagg.isNull = false;
}
else if (inputColVector.compareTo(myagg.value, i) <OperatorSymbol> 0) {
- inputColVector.pisaTimestampUpdate(myagg.value, i);
+ inputColVector.timestampUpdate(myagg.value, i);
}
}
}
@@ -396,13 +397,13 @@ public class <ClassName> extends VectorAggregateExpression {
TimestampColumnVector inputColVector,
int batchSize) {
if (myagg.isNull) {
- inputColVector.pisaTimestampUpdate(myagg.value, 0);
+ inputColVector.timestampUpdate(myagg.value, 0);
myagg.isNull = false;
}
for (int i=0;i<batchSize;++i) {
if (inputColVector.compareTo(myagg.value, i) <OperatorSymbol> 0) {
- inputColVector.pisaTimestampUpdate(myagg.value, i);
+ inputColVector.timestampUpdate(myagg.value, i);
}
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampUtils.java
index 5de055c..bb795fa 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampUtils.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hive.ql.exec.vector;
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.hive.serde2.io.DateWritable;
+import org.apache.hadoop.hive.serde2.io.HiveIntervalDayTimeWritable;
import org.apache.hadoop.hive.serde2.io.TimestampWritable;
public final class TimestampUtils {
@@ -38,4 +39,11 @@ public final class TimestampUtils {
timestampWritable.set(timestampColVector.asScratchTimestamp(elementNum));
return timestampWritable;
}
+
+ public static HiveIntervalDayTimeWritable intervalDayTimeColumnVectorWritable(
+ IntervalDayTimeColumnVector intervalDayTimeColVector, int elementNum,
+ HiveIntervalDayTimeWritable intervalDayTimeWritable) {
+ intervalDayTimeWritable.set(intervalDayTimeColVector.asScratchIntervalDayTime(elementNum));
+ return intervalDayTimeWritable;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAssignRow.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAssignRow.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAssignRow.java
index 965c027..de0300a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAssignRow.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAssignRow.java
@@ -288,7 +288,26 @@ public abstract class VectorAssignRow {
}
}
- private class IntervalDayTimeAssigner extends AbstractTimestampAssigner {
+ private abstract class AbstractIntervalDayTimeAssigner extends Assigner {
+
+ protected IntervalDayTimeColumnVector colVector;
+
+ AbstractIntervalDayTimeAssigner(int columnIndex) {
+ super(columnIndex);
+ }
+
+ @Override
+ void setColumnVector(VectorizedRowBatch batch) {
+ colVector = (IntervalDayTimeColumnVector) batch.cols[columnIndex];
+ }
+
+ @Override
+ void forgetColumnVector() {
+ colVector = null;
+ }
+ }
+
+ private class IntervalDayTimeAssigner extends AbstractIntervalDayTimeAssigner {
IntervalDayTimeAssigner(int columnIndex) {
super(columnIndex);
@@ -301,7 +320,7 @@ public abstract class VectorAssignRow {
} else {
HiveIntervalDayTimeWritable idtw = (HiveIntervalDayTimeWritable) object;
HiveIntervalDayTime idt = idtw.getHiveIntervalDayTime();
- colVector.set(batchIndex, idt.pisaTimestampUpdate(colVector.useScratchPisaTimestamp()));
+ colVector.set(batchIndex, idt);
colVector.isNull[batchIndex] = false;
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnAssignFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnAssignFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnAssignFactory.java
index 463c8a6..96b8f78 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnAssignFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnAssignFactory.java
@@ -25,6 +25,7 @@ import java.util.Map;
import org.apache.hadoop.hive.common.type.HiveChar;
import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
import org.apache.hadoop.hive.common.type.HiveVarchar;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.serde2.io.ByteWritable;
@@ -176,6 +177,16 @@ public class VectorColumnAssignFactory {
}
}
+ private static abstract class VectorIntervalDayTimeColumnAssign
+ extends VectorColumnAssignVectorBase<IntervalDayTimeColumnVector> {
+
+ protected void assignIntervalDayTime(HiveIntervalDayTime value, int index) {
+ outCol.set(index, value);
+ }
+ protected void assignIntervalDayTime(HiveIntervalDayTimeWritable tw, int index) {
+ outCol.set(index, tw.getHiveIntervalDayTime());
+ }
+ }
public static VectorColumnAssign[] buildAssigners(VectorizedRowBatch outputBatch)
throws HiveException {
@@ -364,7 +375,7 @@ public class VectorColumnAssignFactory {
}
}.init(outputBatch, (LongColumnVector) destCol);
break;
- case INTERVAL_DAY_TIME:outVCA = new VectorLongColumnAssign() {
+ case INTERVAL_DAY_TIME:outVCA = new VectorIntervalDayTimeColumnAssign() {
@Override
public void assignObjectValue(Object val, int destIndex) throws HiveException {
if (val == null) {
@@ -372,12 +383,12 @@ public class VectorColumnAssignFactory {
}
else {
HiveIntervalDayTimeWritable bw = (HiveIntervalDayTimeWritable) val;
- assignLong(
- DateUtils.getIntervalDayTimeTotalNanos(bw.getHiveIntervalDayTime()),
+ assignIntervalDayTime(
+ bw.getHiveIntervalDayTime(),
destIndex);
}
}
- }.init(outputBatch, (LongColumnVector) destCol);
+ }.init(outputBatch, (IntervalDayTimeColumnVector) destCol);
break;
default:
throw new HiveException("Incompatible Long vector column and primitive category " +
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnSetInfo.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnSetInfo.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnSetInfo.java
index 0949145..935b47b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnSetInfo.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnSetInfo.java
@@ -60,6 +60,11 @@ public class VectorColumnSetInfo {
protected int[] timestampIndices;
/**
+ * indices of INTERVAL_DAY_TIME primitive keys.
+ */
+ protected int[] intervalDayTimeIndices;
+
+ /**
* Helper class for looking up a key value based on key index.
*/
public class KeyLookupHelper {
@@ -68,12 +73,13 @@ public class VectorColumnSetInfo {
public int stringIndex;
public int decimalIndex;
public int timestampIndex;
+ public int intervalDayTimeIndex;
private static final int INDEX_UNUSED = -1;
private void resetIndices() {
this.longIndex = this.doubleIndex = this.stringIndex = this.decimalIndex =
- timestampIndex = INDEX_UNUSED;
+ timestampIndex = intervalDayTimeIndex = INDEX_UNUSED;
}
public void setLong(int index) {
resetIndices();
@@ -99,6 +105,11 @@ public class VectorColumnSetInfo {
resetIndices();
this.timestampIndex= index;
}
+
+ public void setIntervalDayTime(int index) {
+ resetIndices();
+ this.intervalDayTimeIndex= index;
+ }
}
/**
@@ -114,6 +125,7 @@ public class VectorColumnSetInfo {
protected int stringIndicesIndex;
protected int decimalIndicesIndex;
protected int timestampIndicesIndex;
+ protected int intervalDayTimeIndicesIndex;
protected VectorColumnSetInfo(int keyCount) {
this.keyCount = keyCount;
@@ -130,6 +142,8 @@ public class VectorColumnSetInfo {
decimalIndicesIndex = 0;
timestampIndices = new int[this.keyCount];
timestampIndicesIndex = 0;
+ intervalDayTimeIndices = new int[this.keyCount];
+ intervalDayTimeIndicesIndex = 0;
indexLookup = new KeyLookupHelper[this.keyCount];
}
@@ -172,6 +186,12 @@ public class VectorColumnSetInfo {
++timestampIndicesIndex;
break;
+ case INTERVAL_DAY_TIME:
+ intervalDayTimeIndices[intervalDayTimeIndicesIndex] = addIndex;
+ indexLookup[addIndex].setIntervalDayTime(intervalDayTimeIndicesIndex);
+ ++intervalDayTimeIndicesIndex;
+ break;
+
default:
throw new HiveException("Unexpected column vector type " + columnVectorType);
}
@@ -185,5 +205,6 @@ public class VectorColumnSetInfo {
stringIndices = Arrays.copyOf(stringIndices, stringIndicesIndex);
decimalIndices = Arrays.copyOf(decimalIndices, decimalIndicesIndex);
timestampIndices = Arrays.copyOf(timestampIndices, timestampIndicesIndex);
+ intervalDayTimeIndices = Arrays.copyOf(intervalDayTimeIndices, intervalDayTimeIndicesIndex);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java
index 73476a3..c8e0284 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java
@@ -223,6 +223,34 @@ public class VectorCopyRow {
}
}
}
+
+ private class IntervalDayTimeCopyRow extends CopyRow {
+
+ IntervalDayTimeCopyRow(int inColumnIndex, int outColumnIndex) {
+ super(inColumnIndex, outColumnIndex);
+ }
+
+ @Override
+ void copy(VectorizedRowBatch inBatch, int inBatchIndex, VectorizedRowBatch outBatch, int outBatchIndex) {
+ IntervalDayTimeColumnVector inColVector = (IntervalDayTimeColumnVector) inBatch.cols[inColumnIndex];
+ IntervalDayTimeColumnVector outColVector = (IntervalDayTimeColumnVector) outBatch.cols[outColumnIndex];
+
+ if (inColVector.isRepeating) {
+ if (inColVector.noNulls || !inColVector.isNull[0]) {
+ outColVector.setElement(outBatchIndex, 0, inColVector);
+ } else {
+ VectorizedBatchUtil.setNullColIsNullValue(outColVector, outBatchIndex);
+ }
+ } else {
+ if (inColVector.noNulls || !inColVector.isNull[inBatchIndex]) {
+ outColVector.setElement(outBatchIndex, inBatchIndex, inColVector);
+ } else {
+ VectorizedBatchUtil.setNullColIsNullValue(outColVector, outBatchIndex);
+ }
+ }
+ }
+ }
+
private CopyRow[] subRowToBatchCopiersByValue;
private CopyRow[] subRowToBatchCopiersByReference;
@@ -250,6 +278,10 @@ public class VectorCopyRow {
copyRowByValue = new TimestampCopyRow(inputColumn, outputColumn);
break;
+ case INTERVAL_DAY_TIME:
+ copyRowByValue = new IntervalDayTimeCopyRow(inputColumn, outputColumn);
+ break;
+
case DOUBLE:
copyRowByValue = new DoubleCopyRow(inputColumn, outputColumn);
break;
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java
index 50881e7..3eadc12 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java
@@ -264,7 +264,14 @@ public final class VectorDeserializeRow<T extends DeserializeRead> {
}
}
- private class IntervalDayTimeReader extends AbstractTimestampReader {
+ private abstract class AbstractIntervalDayTimeReader extends Reader<T> {
+
+ AbstractIntervalDayTimeReader(int columnIndex) {
+ super(columnIndex);
+ }
+ }
+
+ private class IntervalDayTimeReader extends AbstractIntervalDayTimeReader {
DeserializeRead.ReadIntervalDayTimeResults readIntervalDayTimeResults;
@@ -275,14 +282,14 @@ public final class VectorDeserializeRow<T extends DeserializeRead> {
@Override
void apply(VectorizedRowBatch batch, int batchIndex) throws IOException {
- TimestampColumnVector colVector = (TimestampColumnVector) batch.cols[columnIndex];
+ IntervalDayTimeColumnVector colVector = (IntervalDayTimeColumnVector) batch.cols[columnIndex];
if (deserializeRead.readCheckNull()) {
VectorizedBatchUtil.setNullColIsNullValue(colVector, batchIndex);
} else {
deserializeRead.readIntervalDayTime(readIntervalDayTimeResults);
HiveIntervalDayTime idt = readIntervalDayTimeResults.getHiveIntervalDayTime();
- colVector.set(batchIndex, idt.pisaTimestampUpdate(colVector.useScratchPisaTimestamp()));
+ colVector.set(batchIndex, idt);
colVector.isNull[batchIndex] = false;
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExpressionDescriptor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExpressionDescriptor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExpressionDescriptor.java
index 0b9ad55..7b3f781 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExpressionDescriptor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExpressionDescriptor.java
@@ -59,6 +59,9 @@ public class VectorExpressionDescriptor {
// TimestampColumnVector -->
// TIMESTAMP
//
+ // IntervalDayTimeColumnVector -->
+ // INTERVAL_DAY_TIME
+ //
public enum ArgumentType {
NONE (0x000),
INT_FAMILY (0x001),
@@ -76,7 +79,6 @@ public class VectorExpressionDescriptor {
INTERVAL_FAMILY (INTERVAL_YEAR_MONTH.value | INTERVAL_DAY_TIME.value),
INT_INTERVAL_YEAR_MONTH (INT_FAMILY.value | INTERVAL_YEAR_MONTH.value),
INT_DATE_INTERVAL_YEAR_MONTH (INT_FAMILY.value | DATE.value | INTERVAL_YEAR_MONTH.value),
- TIMESTAMP_INTERVAL_DAY_TIME (TIMESTAMP.value | INTERVAL_DAY_TIME.value),
STRING_DATETIME_FAMILY (STRING_FAMILY.value | DATETIME_FAMILY.value),
ALL_FAMILY (0xFFF);
@@ -346,7 +348,7 @@ public class VectorExpressionDescriptor {
return ve;
}
} catch (Exception ex) {
- throw new HiveException(ex);
+ throw new HiveException("Could not instantiate VectorExpression class " + ve.getSimpleName(), ex);
}
}
if (LOG.isDebugEnabled()) {
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java
index 622f4a3..e883f38 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java
@@ -32,7 +32,6 @@ import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
import org.apache.hadoop.hive.common.type.HiveVarchar;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.serde2.io.DateWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -323,7 +322,26 @@ public abstract class VectorExtractRow {
}
}
- private class IntervalDayTimeExtractor extends AbstractTimestampExtractor {
+ private abstract class AbstractIntervalDayTimeExtractor extends Extractor {
+
+ protected IntervalDayTimeColumnVector colVector;
+
+ AbstractIntervalDayTimeExtractor(int columnIndex) {
+ super(columnIndex);
+ }
+
+ @Override
+ void setColumnVector(VectorizedRowBatch batch) {
+ colVector = (IntervalDayTimeColumnVector) batch.cols[columnIndex];
+ }
+
+ @Override
+ void forgetColumnVector() {
+ colVector = null;
+ }
+ }
+
+ private class IntervalDayTimeExtractor extends AbstractIntervalDayTimeExtractor {
private HiveIntervalDayTime hiveIntervalDayTime;
@@ -337,7 +355,7 @@ public abstract class VectorExtractRow {
Object extract(int batchIndex) {
int adjustedIndex = (colVector.isRepeating ? 0 : batchIndex);
if (colVector.noNulls || !colVector.isNull[adjustedIndex]) {
- hiveIntervalDayTime.set(colVector.asScratchPisaTimestamp(adjustedIndex));
+ hiveIntervalDayTime.set(colVector.asScratchIntervalDayTime(adjustedIndex));
PrimitiveObjectInspectorFactory.writableHiveIntervalDayTimeObjectInspector.set(object, hiveIntervalDayTime);
return object;
} else {
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupKeyHelper.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupKeyHelper.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupKeyHelper.java
index 9f0ac11..50d0452 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupKeyHelper.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupKeyHelper.java
@@ -131,5 +131,17 @@ public class VectorGroupKeyHelper extends VectorColumnSetInfo {
outputColumnVector.isNull[outputBatch.size] = true;
}
}
+ for(int i=0;i<intervalDayTimeIndices.length; ++i) {
+ int keyIndex = intervalDayTimeIndices[i];
+ IntervalDayTimeColumnVector inputColumnVector = (IntervalDayTimeColumnVector) inputBatch.cols[keyIndex];
+ IntervalDayTimeColumnVector outputColumnVector = (IntervalDayTimeColumnVector) outputBatch.cols[keyIndex];
+ if (inputColumnVector.noNulls || !inputColumnVector.isNull[0]) {
+
+ outputColumnVector.setElement(outputBatch.size, 0, inputColumnVector);
+ } else {
+ outputColumnVector.noNulls = false;
+ outputColumnVector.isNull[outputBatch.size] = true;
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java
index b5d8164..8a101a6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java
@@ -18,16 +18,16 @@
package org.apache.hadoop.hive.ql.exec.vector;
+import java.sql.Timestamp;
import java.util.Arrays;
-import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
-import org.apache.hadoop.hive.serde2.io.TimestampWritable;
import org.apache.hadoop.hive.common.type.HiveDecimal;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
import org.apache.hadoop.hive.ql.exec.KeyWrapper;
import org.apache.hadoop.hive.ql.exec.vector.expressions.StringExpr;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.util.JavaDataModel;
+import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
/**
@@ -44,7 +44,8 @@ public class VectorHashKeyWrapper extends KeyWrapper {
private static final double[] EMPTY_DOUBLE_ARRAY = new double[0];
private static final byte[][] EMPTY_BYTES_ARRAY = new byte[0][];
private static final HiveDecimalWritable[] EMPTY_DECIMAL_ARRAY = new HiveDecimalWritable[0];
- private static final PisaTimestamp[] EMPTY_TIMESTAMP_ARRAY = new PisaTimestamp[0];
+ private static final Timestamp[] EMPTY_TIMESTAMP_ARRAY = new Timestamp[0];
+ private static final HiveIntervalDayTime[] EMPTY_INTERVAL_DAY_TIME_ARRAY = new HiveIntervalDayTime[0];
private long[] longValues;
private double[] doubleValues;
@@ -55,17 +56,21 @@ public class VectorHashKeyWrapper extends KeyWrapper {
private HiveDecimalWritable[] decimalValues;
- private PisaTimestamp[] timestampValues;
+ private Timestamp[] timestampValues;
+
+ private HiveIntervalDayTime[] intervalDayTimeValues;
private boolean[] isNull;
private int hashcode;
public VectorHashKeyWrapper(int longValuesCount, int doubleValuesCount,
- int byteValuesCount, int decimalValuesCount, int timestampValuesCount) {
+ int byteValuesCount, int decimalValuesCount, int timestampValuesCount,
+ int intervalDayTimeValuesCount) {
longValues = longValuesCount > 0 ? new long[longValuesCount] : EMPTY_LONG_ARRAY;
doubleValues = doubleValuesCount > 0 ? new double[doubleValuesCount] : EMPTY_DOUBLE_ARRAY;
decimalValues = decimalValuesCount > 0 ? new HiveDecimalWritable[decimalValuesCount] : EMPTY_DECIMAL_ARRAY;
- timestampValues = timestampValuesCount > 0 ? new PisaTimestamp[timestampValuesCount] : EMPTY_TIMESTAMP_ARRAY;
+ timestampValues = timestampValuesCount > 0 ? new Timestamp[timestampValuesCount] : EMPTY_TIMESTAMP_ARRAY;
+ intervalDayTimeValues = intervalDayTimeValuesCount > 0 ? new HiveIntervalDayTime[intervalDayTimeValuesCount] : EMPTY_INTERVAL_DAY_TIME_ARRAY;
for(int i = 0; i < decimalValuesCount; ++i) {
decimalValues[i] = new HiveDecimalWritable(HiveDecimal.ZERO);
}
@@ -79,10 +84,13 @@ public class VectorHashKeyWrapper extends KeyWrapper {
byteLengths = EMPTY_INT_ARRAY;
}
for(int i = 0; i < timestampValuesCount; ++i) {
- timestampValues[i] = new PisaTimestamp();
+ timestampValues[i] = new Timestamp(0);
+ }
+ for(int i = 0; i < intervalDayTimeValuesCount; ++i) {
+ intervalDayTimeValues[i] = new HiveIntervalDayTime();
}
isNull = new boolean[longValuesCount + doubleValuesCount + byteValuesCount +
- decimalValuesCount + timestampValuesCount];
+ decimalValuesCount + timestampValuesCount + intervalDayTimeValuesCount];
hashcode = 0;
}
@@ -108,6 +116,10 @@ public class VectorHashKeyWrapper extends KeyWrapper {
hashcode ^= timestampValues[i].hashCode();
}
+ for (int i = 0; i < intervalDayTimeValues.length; i++) {
+ hashcode ^= intervalDayTimeValues[i].hashCode();
+ }
+
// This code, with branches and all, is not executed if there are no string keys
for (int i = 0; i < byteValues.length; ++i) {
/*
@@ -146,6 +158,7 @@ public class VectorHashKeyWrapper extends KeyWrapper {
Arrays.equals(doubleValues, keyThat.doubleValues) &&
Arrays.equals(decimalValues, keyThat.decimalValues) &&
Arrays.equals(timestampValues, keyThat.timestampValues) &&
+ Arrays.equals(intervalDayTimeValues, keyThat.intervalDayTimeValues) &&
Arrays.equals(isNull, keyThat.isNull) &&
byteValues.length == keyThat.byteValues.length &&
(0 == byteValues.length || bytesEquals(keyThat));
@@ -212,14 +225,21 @@ public class VectorHashKeyWrapper extends KeyWrapper {
clone.byteLengths = EMPTY_INT_ARRAY;
}
if (timestampValues.length > 0) {
- clone.timestampValues = new PisaTimestamp[timestampValues.length];
+ clone.timestampValues = new Timestamp[timestampValues.length];
for(int i = 0; i < timestampValues.length; ++i) {
- clone.timestampValues[i] = new PisaTimestamp();
- clone.timestampValues[i].update(timestampValues[i]);
+ clone.timestampValues[i] = (Timestamp) timestampValues[i].clone();
}
} else {
clone.timestampValues = EMPTY_TIMESTAMP_ARRAY;
}
+ if (intervalDayTimeValues.length > 0) {
+ clone.intervalDayTimeValues = new HiveIntervalDayTime[intervalDayTimeValues.length];
+ for(int i = 0; i < intervalDayTimeValues.length; ++i) {
+ clone.intervalDayTimeValues[i] = (HiveIntervalDayTime) intervalDayTimeValues[i].clone();
+ }
+ } else {
+ clone.intervalDayTimeValues = EMPTY_INTERVAL_DAY_TIME_ARRAY;
+ }
clone.hashcode = hashcode;
assert clone.equals(this);
@@ -281,14 +301,14 @@ public class VectorHashKeyWrapper extends KeyWrapper {
isNull[longValues.length + doubleValues.length + byteValues.length + index] = true;
}
- public void assignTimestamp(int index, PisaTimestamp value) {
- timestampValues[index].update(value);
+ public void assignTimestamp(int index, Timestamp value) {
+ timestampValues[index] = value;
isNull[longValues.length + doubleValues.length + byteValues.length +
decimalValues.length + index] = false;
}
public void assignTimestamp(int index, TimestampColumnVector colVector, int elementNum) {
- colVector.pisaTimestampUpdate(timestampValues[index], elementNum);
+ colVector.timestampUpdate(timestampValues[index], elementNum);
isNull[longValues.length + doubleValues.length + byteValues.length +
decimalValues.length + index] = false;
}
@@ -298,15 +318,33 @@ public class VectorHashKeyWrapper extends KeyWrapper {
decimalValues.length + index] = true;
}
+ public void assignIntervalDayTime(int index, HiveIntervalDayTime value) {
+ intervalDayTimeValues[index].set(value);
+ isNull[longValues.length + doubleValues.length + byteValues.length +
+ decimalValues.length + timestampValues.length + index] = false;
+ }
+
+ public void assignIntervalDayTime(int index, IntervalDayTimeColumnVector colVector, int elementNum) {
+ intervalDayTimeValues[index].set(colVector.asScratchIntervalDayTime(elementNum));
+ isNull[longValues.length + doubleValues.length + byteValues.length +
+ decimalValues.length + timestampValues.length + index] = false;
+ }
+
+ public void assignNullIntervalDayTime(int index) {
+ isNull[longValues.length + doubleValues.length + byteValues.length +
+ decimalValues.length + timestampValues.length + index] = true;
+ }
+
@Override
public String toString()
{
- return String.format("%d[%s] %d[%s] %d[%s] %d[%s] %d[%s]",
+ return String.format("%d[%s] %d[%s] %d[%s] %d[%s] %d[%s] %d[%s]",
longValues.length, Arrays.toString(longValues),
doubleValues.length, Arrays.toString(doubleValues),
byteValues.length, Arrays.toString(byteValues),
decimalValues.length, Arrays.toString(decimalValues),
- timestampValues.length, Arrays.toString(timestampValues));
+ timestampValues.length, Arrays.toString(timestampValues),
+ intervalDayTimeValues.length, Arrays.toString(intervalDayTimeValues));
}
public boolean getIsLongNull(int i) {
@@ -364,9 +402,17 @@ public class VectorHashKeyWrapper extends KeyWrapper {
decimalValues.length + i];
}
- public PisaTimestamp getTimestamp(int i) {
+ public Timestamp getTimestamp(int i) {
return timestampValues[i];
}
+ public boolean getIsIntervalDayTimeNull(int i) {
+ return isNull[longValues.length + doubleValues.length + byteValues.length +
+ decimalValues.length + timestampValues.length + i];
+ }
+
+ public HiveIntervalDayTime getIntervalDayTime(int i) {
+ return intervalDayTimeValues[i];
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java
index 1c34124..bfd26ae 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java
@@ -198,6 +198,28 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
columnVector.noNulls, columnVector.isRepeating, batch.selectedInUse));
}
}
+ for(int i=0;i<intervalDayTimeIndices.length; ++i) {
+ int keyIndex = intervalDayTimeIndices[i];
+ int columnIndex = keyExpressions[keyIndex].getOutputColumn();
+ IntervalDayTimeColumnVector columnVector = (IntervalDayTimeColumnVector) batch.cols[columnIndex];
+ if (columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
+ assignIntervalDayTimeNoNullsNoRepeatingNoSelection(i, batch.size, columnVector);
+ } else if (columnVector.noNulls && !columnVector.isRepeating && batch.selectedInUse) {
+ assignIntervalDayTimeNoNullsNoRepeatingSelection(i, batch.size, columnVector, batch.selected);
+ } else if (columnVector.noNulls && columnVector.isRepeating) {
+ assignIntervalDayTimeNoNullsRepeating(i, batch.size, columnVector);
+ } else if (!columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
+ assignIntervalDayTimeNullsNoRepeatingNoSelection(i, batch.size, columnVector);
+ } else if (!columnVector.noNulls && columnVector.isRepeating) {
+ assignIntervalDayTimeNullsRepeating(i, batch.size, columnVector);
+ } else if (!columnVector.noNulls && !columnVector.isRepeating && batch.selectedInUse) {
+ assignIntervalDayTimeNullsNoRepeatingSelection (i, batch.size, columnVector, batch.selected);
+ } else {
+ throw new HiveException (String.format(
+ "Unimplemented intervalDayTime null/repeat/selected combination %b/%b/%b",
+ columnVector.noNulls, columnVector.isRepeating, batch.selectedInUse));
+ }
+ }
for(int i=0;i<batch.size;++i) {
vectorHashKeyWrappers[i].setHashKey();
}
@@ -596,6 +618,81 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
vectorHashKeyWrappers[r].assignTimestamp(index, columnVector, r);
}
}
+
+ /**
+ * Helper method to assign values from a vector column into the key wrapper.
+ * Optimized for IntervalDayTime type, possible nulls, no repeat values, batch selection vector.
+ */
+ private void assignIntervalDayTimeNullsNoRepeatingSelection(int index, int size,
+ IntervalDayTimeColumnVector columnVector, int[] selected) {
+ for(int i = 0; i < size; ++i) {
+ int row = selected[i];
+ if (!columnVector.isNull[row]) {
+ vectorHashKeyWrappers[i].assignIntervalDayTime(index, columnVector, row);
+ } else {
+ vectorHashKeyWrappers[i].assignNullIntervalDayTime(index);
+ }
+ }
+ }
+
+ /**
+ * Helper method to assign values from a vector column into the key wrapper.
+ * Optimized for IntervalDayTime type, repeat null values.
+ */
+ private void assignIntervalDayTimeNullsRepeating(int index, int size,
+ IntervalDayTimeColumnVector columnVector) {
+ for(int r = 0; r < size; ++r) {
+ vectorHashKeyWrappers[r].assignNullIntervalDayTime(index);
+ }
+ }
+
+ /**
+ * Helper method to assign values from a vector column into the key wrapper.
+ * Optimized for IntervalDayTime type, possible nulls, repeat values.
+ */
+ private void assignIntervalDayTimeNullsNoRepeatingNoSelection(int index, int size,
+ IntervalDayTimeColumnVector columnVector) {
+ for(int r = 0; r < size; ++r) {
+ if (!columnVector.isNull[r]) {
+ vectorHashKeyWrappers[r].assignIntervalDayTime(index, columnVector, r);
+ } else {
+ vectorHashKeyWrappers[r].assignNullIntervalDayTime(index);
+ }
+ }
+ }
+
+ /**
+ * Helper method to assign values from a vector column into the key wrapper.
+ * Optimized for IntervalDayTime type, no nulls, repeat values, no selection vector.
+ */
+ private void assignIntervalDayTimeNoNullsRepeating(int index, int size, IntervalDayTimeColumnVector columnVector) {
+ for(int r = 0; r < size; ++r) {
+ vectorHashKeyWrappers[r].assignIntervalDayTime(index, columnVector, 0);
+ }
+ }
+
+ /**
+ * Helper method to assign values from a vector column into the key wrapper.
+ * Optimized for IntervalDayTime type, no nulls, no repeat values, batch selection vector.
+ */
+ private void assignIntervalDayTimeNoNullsNoRepeatingSelection(int index, int size,
+ IntervalDayTimeColumnVector columnVector, int[] selected) {
+ for(int r = 0; r < size; ++r) {
+ vectorHashKeyWrappers[r].assignIntervalDayTime(index, columnVector, selected[r]);
+ }
+ }
+
+ /**
+ * Helper method to assign values from a vector column into the key wrapper.
+ * Optimized for IntervalDayTime type, no nulls, no repeat values, no selection vector.
+ */
+ private void assignIntervalDayTimeNoNullsNoRepeatingNoSelection(int index, int size,
+ IntervalDayTimeColumnVector columnVector) {
+ for(int r = 0; r < size; ++r) {
+ vectorHashKeyWrappers[r].assignIntervalDayTime(index, columnVector, r);
+ }
+ }
+
/**
* Prepares a VectorHashKeyWrapperBatch to work for a specific set of keys.
* Computes the fast access lookup indices, preallocates all needed internal arrays.
@@ -638,6 +735,7 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
compiledKeyWrapperBatch.keysFixedSize += model.lengthForObjectArrayOfSize(compiledKeyWrapperBatch.stringIndices.length);
compiledKeyWrapperBatch.keysFixedSize += model.lengthForObjectArrayOfSize(compiledKeyWrapperBatch.decimalIndices.length);
compiledKeyWrapperBatch.keysFixedSize += model.lengthForObjectArrayOfSize(compiledKeyWrapperBatch.timestampIndices.length);
+ compiledKeyWrapperBatch.keysFixedSize += model.lengthForObjectArrayOfSize(compiledKeyWrapperBatch.intervalDayTimeIndices.length);
compiledKeyWrapperBatch.keysFixedSize += model.lengthForIntArrayOfSize(compiledKeyWrapperBatch.longIndices.length) * 2;
compiledKeyWrapperBatch.keysFixedSize +=
model.lengthForBooleanArrayOfSize(keyExpressions.length);
@@ -647,7 +745,8 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
public VectorHashKeyWrapper allocateKeyWrapper() {
return new VectorHashKeyWrapper(longIndices.length, doubleIndices.length,
- stringIndices.length, decimalIndices.length, timestampIndices.length);
+ stringIndices.length, decimalIndices.length, timestampIndices.length,
+ intervalDayTimeIndices.length);
}
/**
@@ -679,12 +778,15 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
return kw.getIsTimestampNull(klh.timestampIndex)? null :
keyOutputWriter.writeValue(
kw.getTimestamp(klh.timestampIndex));
- }
- else {
+ } else if (klh.intervalDayTimeIndex >= 0) {
+ return kw.getIsIntervalDayTimeNull(klh.intervalDayTimeIndex)? null :
+ keyOutputWriter.writeValue(
+ kw.getIntervalDayTime(klh.intervalDayTimeIndex));
+ } else {
throw new HiveException(String.format(
- "Internal inconsistent KeyLookupHelper at index [%d]:%d %d %d %d %d",
+ "Internal inconsistent KeyLookupHelper at index [%d]:%d %d %d %d %d %d",
i, klh.longIndex, klh.doubleIndex, klh.stringIndex, klh.decimalIndex,
- klh.timestampIndex));
+ klh.timestampIndex, klh.intervalDayTimeIndex));
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSerializeRow.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSerializeRow.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSerializeRow.java
index dea38e8..6af3d99 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSerializeRow.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSerializeRow.java
@@ -23,7 +23,6 @@ import java.sql.Timestamp;
import java.util.List;
import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.serde2.ByteStream.Output;
import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
@@ -332,11 +331,11 @@ public final class VectorSerializeRow<T extends SerializeWrite> {
@Override
boolean apply(VectorizedRowBatch batch, int batchIndex) throws IOException {
- TimestampColumnVector colVector = (TimestampColumnVector) batch.cols[columnIndex];
+ IntervalDayTimeColumnVector colVector = (IntervalDayTimeColumnVector) batch.cols[columnIndex];
if (colVector.isRepeating) {
if (colVector.noNulls || !colVector.isNull[0]) {
- hiveIntervalDayTime.set(colVector.asScratchPisaTimestamp(0));
+ hiveIntervalDayTime.set(colVector.asScratchIntervalDayTime(0));
serializeWrite.writeHiveIntervalDayTime(hiveIntervalDayTime);
return true;
} else {
@@ -345,7 +344,7 @@ public final class VectorSerializeRow<T extends SerializeWrite> {
}
} else {
if (colVector.noNulls || !colVector.isNull[batchIndex]) {
- hiveIntervalDayTime.set(colVector.asScratchPisaTimestamp(batchIndex));
+ hiveIntervalDayTime.set(colVector.asScratchIntervalDayTime(batchIndex));
serializeWrite.writeHiveIntervalDayTime(hiveIntervalDayTime);
return true;
} else {
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
index 3f95be2..0552f9d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
@@ -68,11 +68,13 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen.VectorUD
import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen.VectorUDAFMaxLong;
import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen.VectorUDAFMaxString;
import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen.VectorUDAFMaxTimestamp;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen.VectorUDAFMaxIntervalDayTime;
import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen.VectorUDAFMinDecimal;
import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen.VectorUDAFMinDouble;
import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen.VectorUDAFMinLong;
import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen.VectorUDAFMinString;
import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen.VectorUDAFMinTimestamp;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen.VectorUDAFMinIntervalDayTime;
import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen.VectorUDAFStdPopDecimal;
import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen.VectorUDAFStdPopDouble;
import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen.VectorUDAFStdPopLong;
@@ -2333,10 +2335,12 @@ public class VectorizationContext {
case INTERVAL_YEAR_MONTH:
return ColumnVector.Type.LONG;
- case INTERVAL_DAY_TIME:
case TIMESTAMP:
return ColumnVector.Type.TIMESTAMP;
+ case INTERVAL_DAY_TIME:
+ return ColumnVector.Type.INTERVAL_DAY_TIME;
+
case FLOAT:
case DOUBLE:
return ColumnVector.Type.DOUBLE;
@@ -2369,19 +2373,20 @@ public class VectorizationContext {
add(new AggregateDefinition("min", VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY, null, VectorUDAFMinDouble.class));
add(new AggregateDefinition("min", VectorExpressionDescriptor.ArgumentType.STRING_FAMILY, null, VectorUDAFMinString.class));
add(new AggregateDefinition("min", VectorExpressionDescriptor.ArgumentType.DECIMAL, null, VectorUDAFMinDecimal.class));
- add(new AggregateDefinition("min", VectorExpressionDescriptor.ArgumentType.TIMESTAMP_INTERVAL_DAY_TIME, null, VectorUDAFMinTimestamp.class));
+ add(new AggregateDefinition("min", VectorExpressionDescriptor.ArgumentType.TIMESTAMP, null, VectorUDAFMinTimestamp.class));
add(new AggregateDefinition("max", VectorExpressionDescriptor.ArgumentType.INT_DATE_INTERVAL_YEAR_MONTH, null, VectorUDAFMaxLong.class));
add(new AggregateDefinition("max", VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY, null, VectorUDAFMaxDouble.class));
add(new AggregateDefinition("max", VectorExpressionDescriptor.ArgumentType.STRING_FAMILY, null, VectorUDAFMaxString.class));
add(new AggregateDefinition("max", VectorExpressionDescriptor.ArgumentType.DECIMAL, null, VectorUDAFMaxDecimal.class));
- add(new AggregateDefinition("max", VectorExpressionDescriptor.ArgumentType.TIMESTAMP_INTERVAL_DAY_TIME, null, VectorUDAFMaxTimestamp.class));
+ add(new AggregateDefinition("max", VectorExpressionDescriptor.ArgumentType.TIMESTAMP, null, VectorUDAFMaxTimestamp.class));
add(new AggregateDefinition("count", VectorExpressionDescriptor.ArgumentType.NONE, GroupByDesc.Mode.HASH, VectorUDAFCountStar.class));
add(new AggregateDefinition("count", VectorExpressionDescriptor.ArgumentType.INT_DATE_INTERVAL_YEAR_MONTH, GroupByDesc.Mode.HASH, VectorUDAFCount.class));
add(new AggregateDefinition("count", VectorExpressionDescriptor.ArgumentType.INT_FAMILY, GroupByDesc.Mode.MERGEPARTIAL, VectorUDAFCountMerge.class));
add(new AggregateDefinition("count", VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY, GroupByDesc.Mode.HASH, VectorUDAFCount.class));
add(new AggregateDefinition("count", VectorExpressionDescriptor.ArgumentType.STRING_FAMILY, GroupByDesc.Mode.HASH, VectorUDAFCount.class));
add(new AggregateDefinition("count", VectorExpressionDescriptor.ArgumentType.DECIMAL, GroupByDesc.Mode.HASH, VectorUDAFCount.class));
- add(new AggregateDefinition("count", VectorExpressionDescriptor.ArgumentType.TIMESTAMP_INTERVAL_DAY_TIME, GroupByDesc.Mode.HASH, VectorUDAFCount.class));
+ add(new AggregateDefinition("count", VectorExpressionDescriptor.ArgumentType.TIMESTAMP, GroupByDesc.Mode.HASH, VectorUDAFCount.class));
+ add(new AggregateDefinition("count", VectorExpressionDescriptor.ArgumentType.INTERVAL_DAY_TIME, GroupByDesc.Mode.HASH, VectorUDAFCount.class));
add(new AggregateDefinition("sum", VectorExpressionDescriptor.ArgumentType.INT_FAMILY, null, VectorUDAFSumLong.class));
add(new AggregateDefinition("sum", VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY, null, VectorUDAFSumDouble.class));
add(new AggregateDefinition("sum", VectorExpressionDescriptor.ArgumentType.DECIMAL, null, VectorUDAFSumDecimal.class));
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
index a68d0cc..be04da8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
@@ -144,9 +144,10 @@ public class VectorizedBatchUtil {
case DATE:
case INTERVAL_YEAR_MONTH:
return new LongColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
- case INTERVAL_DAY_TIME:
case TIMESTAMP:
return new TimestampColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
+ case INTERVAL_DAY_TIME:
+ return new IntervalDayTimeColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
case FLOAT:
case DOUBLE:
return new DoubleColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
@@ -417,14 +418,14 @@ public class VectorizedBatchUtil {
}
break;
case INTERVAL_DAY_TIME: {
- LongColumnVector lcv = (LongColumnVector) batch.cols[offset + colIndex];
+ IntervalDayTimeColumnVector icv = (IntervalDayTimeColumnVector) batch.cols[offset + colIndex];
if (writableCol != null) {
- HiveIntervalDayTime i = ((HiveIntervalDayTimeWritable) writableCol).getHiveIntervalDayTime();
- lcv.vector[rowIndex] = DateUtils.getIntervalDayTimeTotalNanos(i);
- lcv.isNull[rowIndex] = false;
+ HiveIntervalDayTime idt = ((HiveIntervalDayTimeWritable) writableCol).getHiveIntervalDayTime();
+ icv.set(rowIndex, idt);
+ icv.isNull[rowIndex] = false;
} else {
- lcv.vector[rowIndex] = 1;
- setNullColIsNullValue(lcv, rowIndex);
+ icv.setNullValue(rowIndex);
+ setNullColIsNullValue(icv, rowIndex);
}
}
break;
@@ -585,6 +586,8 @@ public class VectorizedBatchUtil {
decColVector.scale);
} else if (source instanceof TimestampColumnVector) {
return new TimestampColumnVector(((TimestampColumnVector) source).getLength());
+ } else if (source instanceof IntervalDayTimeColumnVector) {
+ return new IntervalDayTimeColumnVector(((IntervalDayTimeColumnVector) source).getLength());
} else if (source instanceof ListColumnVector) {
ListColumnVector src = (ListColumnVector) source;
ColumnVector child = cloneColumnVector(src.child);
@@ -688,6 +691,9 @@ public class VectorizedBatchUtil {
Timestamp timestamp = new Timestamp(0);
((TimestampColumnVector) colVector).timestampUpdate(timestamp, index);
sb.append(timestamp.toString());
+ } else if (colVector instanceof IntervalDayTimeColumnVector) {
+ HiveIntervalDayTime intervalDayTime = ((IntervalDayTimeColumnVector) colVector).asScratchIntervalDayTime(index);
+ sb.append(intervalDayTime.toString());
} else {
sb.append("Unknown");
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java
index 7e79e1e..0724191 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java
@@ -400,14 +400,14 @@ public class VectorizedRowBatchCtx {
}
case INTERVAL_DAY_TIME: {
- TimestampColumnVector tcv = (TimestampColumnVector) batch.cols[colIndex];
+ IntervalDayTimeColumnVector icv = (IntervalDayTimeColumnVector) batch.cols[colIndex];
if (value == null) {
- tcv.noNulls = false;
- tcv.isNull[0] = true;
- tcv.isRepeating = true;
+ icv.noNulls = false;
+ icv.isNull[0] = true;
+ icv.isRepeating = true;
} else {
- tcv.fill(((HiveIntervalDayTime) value).pisaTimestampUpdate(tcv.useScratchPisaTimestamp()));
- tcv.isNull[0] = false;
+ icv.fill(((HiveIntervalDayTime) value));
+ icv.isNull[0] = false;
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDecimalToTimestamp.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDecimalToTimestamp.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDecimalToTimestamp.java
index 2b0068d..6225ade 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDecimalToTimestamp.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDecimalToTimestamp.java
@@ -44,7 +44,6 @@ public class CastDecimalToTimestamp extends FuncDecimalToTimestamp {
@Override
protected void func(TimestampColumnVector outV, DecimalColumnVector inV, int i) {
- Timestamp timestamp = TimestampWritable.decimalToTimestamp(inV.vector[i].getHiveDecimal());
- outV.set(i, timestamp);
+ outV.set(i, TimestampWritable.decimalToTimestamp(inV.vector[i].getHiveDecimal()));
}
}
[33/50] [abbrv] hive git commit: HIVE-13111: Fix timestamp /
interval_day_time wrong results with HIVE-9862 (Matt McCline,
reviewed by Jason Dere)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampColumnCompareTimestampScalar.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampColumnCompareTimestampScalar.txt b/ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampColumnCompareTimestampScalar.txt
index f744d9b..bab8508 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampColumnCompareTimestampScalar.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampColumnCompareTimestampScalar.txt
@@ -20,24 +20,130 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
import java.sql.Timestamp;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
-
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
+import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.*;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
/**
- * Generated from template FilterTimestampColumnCompareScalar.txt, which covers comparison
- * expressions between a datetime/interval column and a scalar of the same type, however output is not
- * produced in a separate column.
+ * Generated from template FilterColumnCompareScalar.txt, which covers binary comparison
+ * expressions between a column and a scalar, however output is not produced in a separate column.
* The selected vector of the input {@link VectorizedRowBatch} is updated for in-place filtering.
*/
-public class <ClassName> extends <BaseClassName> {
+public class <ClassName> extends VectorExpression {
+
+ private static final long serialVersionUID = 1L;
+
+ private int colNum;
+ private <HiveOperandType> value;
- public <ClassName>(int colNum, Timestamp value) {
- super(colNum, new PisaTimestamp(value));
+ public <ClassName>(int colNum, <HiveOperandType> value) {
+ this.colNum = colNum;
+ this.value = value;
}
public <ClassName>() {
- super();
+ }
+
+ @Override
+ public void evaluate(VectorizedRowBatch batch) {
+
+ if (childExpressions != null) {
+ super.evaluateChildren(batch);
+ }
+
+ // Input #1 is type <OperandType>.
+ <InputColumnVectorType> inputColVector1 = (<InputColumnVectorType>) batch.cols[colNum];
+
+ int[] sel = batch.selected;
+ boolean[] nullPos = inputColVector1.isNull;
+ int n = batch.size;
+
+ // return immediately if batch is empty
+ if (n == 0) {
+ return;
+ }
+
+ if (inputColVector1.noNulls) {
+ if (inputColVector1.isRepeating) {
+ //All must be selected otherwise size would be zero
+ //Repeating property will not change.
+ if (!(inputColVector1.compareTo(0, value) <OperatorSymbol> 0)) {
+ //Entire batch is filtered out.
+ batch.size = 0;
+ }
+ } else if (batch.selectedInUse) {
+ int newSize = 0;
+ for(int j=0; j != n; j++) {
+ int i = sel[j];
+ if (inputColVector1.compareTo(i, value) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ batch.size = newSize;
+ } else {
+ int newSize = 0;
+ for(int i = 0; i != n; i++) {
+ if (inputColVector1.compareTo(i, value) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ if (newSize < n) {
+ batch.size = newSize;
+ batch.selectedInUse = true;
+ }
+ }
+ } else {
+ if (inputColVector1.isRepeating) {
+ //All must be selected otherwise size would be zero
+ //Repeating property will not change.
+ if (!nullPos[0]) {
+ if (!(inputColVector1.compareTo(0, value) <OperatorSymbol> 0)) {
+ //Entire batch is filtered out.
+ batch.size = 0;
+ }
+ } else {
+ batch.size = 0;
+ }
+ } else if (batch.selectedInUse) {
+ int newSize = 0;
+ for(int j=0; j != n; j++) {
+ int i = sel[j];
+ if (!nullPos[i]) {
+ if (inputColVector1.compareTo(i, value) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ }
+ //Change the selected vector
+ batch.size = newSize;
+ } else {
+ int newSize = 0;
+ for(int i = 0; i != n; i++) {
+ if (!nullPos[i]) {
+ if (inputColVector1.compareTo(i, value) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ }
+ if (newSize < n) {
+ batch.size = newSize;
+ batch.selectedInUse = true;
+ }
+ }
+ }
+ }
+
+ @Override
+ public int getOutputColumn() {
+ return -1;
+ }
+
+ @Override
+ public String getOutputType() {
+ return "boolean";
}
@Override
@@ -47,8 +153,8 @@ public class <ClassName> extends <BaseClassName> {
VectorExpressionDescriptor.Mode.FILTER)
.setNumArguments(2)
.setArgumentTypes(
- VectorExpressionDescriptor.ArgumentType.getType("timestamp"),
- VectorExpressionDescriptor.ArgumentType.getType("timestamp"))
+ VectorExpressionDescriptor.ArgumentType.getType("<OperandType>"),
+ VectorExpressionDescriptor.ArgumentType.getType("<OperandType>"))
.setInputExpressionTypes(
VectorExpressionDescriptor.InputExpressionType.COLUMN,
VectorExpressionDescriptor.InputExpressionType.SCALAR).build();
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampColumnCompareTimestampScalarBase.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampColumnCompareTimestampScalarBase.txt b/ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampColumnCompareTimestampScalarBase.txt
deleted file mode 100644
index c84b4bf..0000000
--- a/ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampColumnCompareTimestampScalarBase.txt
+++ /dev/null
@@ -1,145 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
-
-import java.sql.Timestamp;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
-
-import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
-import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
-import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
-
-/**
- * Generated from template FilterColumnCompareScalar.txt, which covers binary comparison
- * expressions between a column and a scalar, however output is not produced in a separate column.
- * The selected vector of the input {@link VectorizedRowBatch} is updated for in-place filtering.
- */
-public abstract class <ClassName> extends VectorExpression {
-
- private static final long serialVersionUID = 1L;
-
- private int colNum;
- private PisaTimestamp value;
-
- public <ClassName>(int colNum, PisaTimestamp value) {
- this.colNum = colNum;
- this.value = value;
- }
-
- public <ClassName>() {
- }
-
- @Override
- public void evaluate(VectorizedRowBatch batch) {
-
- if (childExpressions != null) {
- super.evaluateChildren(batch);
- }
-
- TimestampColumnVector inputColVector = (TimestampColumnVector) batch.cols[colNum];
- int[] sel = batch.selected;
- boolean[] nullPos = inputColVector.isNull;
- int n = batch.size;
-
- // return immediately if batch is empty
- if (n == 0) {
- return;
- }
-
- if (inputColVector.noNulls) {
- if (inputColVector.isRepeating) {
- //All must be selected otherwise size would be zero
- //Repeating property will not change.
- if (!(inputColVector.compareTo(0, value) <OperatorSymbol> 0)) {
- //Entire batch is filtered out.
- batch.size = 0;
- }
- } else if (batch.selectedInUse) {
- int newSize = 0;
- for(int j=0; j != n; j++) {
- int i = sel[j];
- if (inputColVector.compareTo(i, value) <OperatorSymbol> 0) {
- sel[newSize++] = i;
- }
- }
- batch.size = newSize;
- } else {
- int newSize = 0;
- for(int i = 0; i != n; i++) {
- if (inputColVector.compareTo(i, value) <OperatorSymbol> 0) {
- sel[newSize++] = i;
- }
- }
- if (newSize < n) {
- batch.size = newSize;
- batch.selectedInUse = true;
- }
- }
- } else {
- if (inputColVector.isRepeating) {
- //All must be selected otherwise size would be zero
- //Repeating property will not change.
- if (!nullPos[0]) {
- if (!(inputColVector.compareTo(0, value) <OperatorSymbol> 0)) {
- //Entire batch is filtered out.
- batch.size = 0;
- }
- } else {
- batch.size = 0;
- }
- } else if (batch.selectedInUse) {
- int newSize = 0;
- for(int j=0; j != n; j++) {
- int i = sel[j];
- if (!nullPos[i]) {
- if (inputColVector.compareTo(i, value) <OperatorSymbol> 0) {
- sel[newSize++] = i;
- }
- }
- }
- //Change the selected vector
- batch.size = newSize;
- } else {
- int newSize = 0;
- for(int i = 0; i != n; i++) {
- if (!nullPos[i]) {
- if (inputColVector.compareTo(i, value) <OperatorSymbol> 0) {
- sel[newSize++] = i;
- }
- }
- }
- if (newSize < n) {
- batch.size = newSize;
- batch.selectedInUse = true;
- }
- }
- }
- }
-
- @Override
- public int getOutputColumn() {
- return -1;
- }
-
- @Override
- public String getOutputType() {
- return "boolean";
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampScalarCompareLongDoubleColumn.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampScalarCompareLongDoubleColumn.txt b/ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampScalarCompareLongDoubleColumn.txt
index c3cd3b4..5e418de 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampScalarCompareLongDoubleColumn.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampScalarCompareLongDoubleColumn.txt
@@ -19,7 +19,6 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
import java.sql.Timestamp;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.*;
@@ -36,7 +35,7 @@ public class <ClassName> extends <BaseClassName> {
private static final long serialVersionUID = 1L;
public <ClassName>(Timestamp value, int colNum) {
- super(new PisaTimestamp(value).<GetTimestampLongDoubleMethod>(), colNum);
+ super(TimestampColumnVector.<GetTimestampLongDoubleMethod>(value), colNum);
}
public <ClassName>() {
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampScalarCompareTimestampColumn.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampScalarCompareTimestampColumn.txt b/ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampScalarCompareTimestampColumn.txt
index 05ab310..ff5d11e 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampScalarCompareTimestampColumn.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampScalarCompareTimestampColumn.txt
@@ -20,24 +20,132 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
import java.sql.Timestamp;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
-
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
+import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.*;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
+import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
+import org.apache.hadoop.hive.common.type.HiveDecimal;
/**
- * Generated from template FilterTimestampScalarCompareTimestampColumn.txt, which covers comparison
- * expressions between a datetime/interval column and a scalar of the same type, however output is not
- * produced in a separate column.
- * The selected vector of the input {@link VectorizedRowBatch} is updated for in-place filtering.
+ * This is a generated class to evaluate a <OperatorSymbol> comparison on a vector of timestamp
+ * values.
*/
-public class <ClassName> extends <BaseClassName> {
+public class <ClassName> extends VectorExpression {
+
+ private static final long serialVersionUID = 1L;
+
+ private int colNum;
+ private <HiveOperandType> value;
- public <ClassName>(Timestamp value, int colNum) {
- super(new PisaTimestamp(value), colNum);
+ public <ClassName>(<HiveOperandType> value, int colNum) {
+ this.colNum = colNum;
+ this.value = value;
}
public <ClassName>() {
- super();
+ }
+
+ @Override
+ public void evaluate(VectorizedRowBatch batch) {
+ if (childExpressions != null) {
+ super.evaluateChildren(batch);
+ }
+ // Input #2 is type <OperandType>.
+ <InputColumnVectorType> inputColVector2 = (<InputColumnVectorType>) batch.cols[colNum];
+
+ int[] sel = batch.selected;
+ boolean[] nullPos = inputColVector2.isNull;
+ int n = batch.size;
+
+ // return immediately if batch is empty
+ if (n == 0) {
+ return;
+ }
+
+ if (inputColVector2.noNulls) {
+ if (inputColVector2.isRepeating) {
+
+ // All must be selected otherwise size would be zero. Repeating property will not change.
+ if (!(inputColVector2.compareTo(value, 0) <OperatorSymbol> 0)) {
+
+ // Entire batch is filtered out.
+ batch.size = 0;
+ }
+ } else if (batch.selectedInUse) {
+ int newSize = 0;
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (inputColVector2.compareTo(value, i) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ batch.size = newSize;
+ } else {
+ int newSize = 0;
+ for(int i = 0; i != n; i++) {
+ if (inputColVector2.compareTo(value, i) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ if (newSize < n) {
+ batch.size = newSize;
+ batch.selectedInUse = true;
+ }
+ }
+ } else {
+ if (inputColVector2.isRepeating) {
+
+ // All must be selected otherwise size would be zero. Repeating property will not change.
+ if (!nullPos[0]) {
+ if (!(inputColVector2.compareTo(value, 0) <OperatorSymbol> 0)) {
+
+ // Entire batch is filtered out.
+ batch.size = 0;
+ }
+ } else {
+ batch.size = 0;
+ }
+ } else if (batch.selectedInUse) {
+ int newSize = 0;
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (!nullPos[i]) {
+ if (inputColVector2.compareTo(value, i) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ }
+
+ // Change the selected vector
+ batch.size = newSize;
+ } else {
+ int newSize = 0;
+ for(int i = 0; i != n; i++) {
+ if (!nullPos[i]) {
+ if (inputColVector2.compareTo(value, i) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ }
+ if (newSize < n) {
+ batch.size = newSize;
+ batch.selectedInUse = true;
+ }
+ }
+ }
+ }
+
+ @Override
+ public int getOutputColumn() {
+ return -1;
+ }
+
+ @Override
+ public String getOutputType() {
+ return "boolean";
}
@Override
@@ -47,8 +155,8 @@ public class <ClassName> extends <BaseClassName> {
VectorExpressionDescriptor.Mode.FILTER)
.setNumArguments(2)
.setArgumentTypes(
- VectorExpressionDescriptor.ArgumentType.getType("timestamp"),
- VectorExpressionDescriptor.ArgumentType.getType("timestamp"))
+ VectorExpressionDescriptor.ArgumentType.getType("<OperandType>"),
+ VectorExpressionDescriptor.ArgumentType.getType("<OperandType>"))
.setInputExpressionTypes(
VectorExpressionDescriptor.InputExpressionType.SCALAR,
VectorExpressionDescriptor.InputExpressionType.COLUMN).build();
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampScalarCompareTimestampColumnBase.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampScalarCompareTimestampColumnBase.txt b/ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampScalarCompareTimestampColumnBase.txt
deleted file mode 100644
index 608faef..0000000
--- a/ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampScalarCompareTimestampColumnBase.txt
+++ /dev/null
@@ -1,147 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
-
-import java.sql.Timestamp;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
-
-import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
-import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
-import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
-import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
-import org.apache.hadoop.hive.common.type.HiveDecimal;
-
-/**
- * This is a generated class to evaluate a <OperatorSymbol> comparison on a vector of timestamp
- * values.
- */
-public abstract class <ClassName> extends VectorExpression {
-
- private static final long serialVersionUID = 1L;
-
- private int colNum;
- private PisaTimestamp value;
-
- public <ClassName>(PisaTimestamp value, int colNum) {
- this.colNum = colNum;
- this.value = value;
- }
-
- public <ClassName>() {
- }
-
- @Override
- public void evaluate(VectorizedRowBatch batch) {
- if (childExpressions != null) {
- super.evaluateChildren(batch);
- }
- TimestampColumnVector inputColVector = (TimestampColumnVector) batch.cols[colNum];
- int[] sel = batch.selected;
- boolean[] nullPos = inputColVector.isNull;
- int n = batch.size;
-
- // return immediately if batch is empty
- if (n == 0) {
- return;
- }
-
- if (inputColVector.noNulls) {
- if (inputColVector.isRepeating) {
-
- // All must be selected otherwise size would be zero. Repeating property will not change.
- if (!(inputColVector.compareTo(value, 0) <OperatorSymbol> 0)) {
-
- // Entire batch is filtered out.
- batch.size = 0;
- }
- } else if (batch.selectedInUse) {
- int newSize = 0;
- for(int j = 0; j != n; j++) {
- int i = sel[j];
- if (inputColVector.compareTo(value, i) <OperatorSymbol> 0) {
- sel[newSize++] = i;
- }
- }
- batch.size = newSize;
- } else {
- int newSize = 0;
- for(int i = 0; i != n; i++) {
- if (inputColVector.compareTo(value, i) <OperatorSymbol> 0) {
- sel[newSize++] = i;
- }
- }
- if (newSize < n) {
- batch.size = newSize;
- batch.selectedInUse = true;
- }
- }
- } else {
- if (inputColVector.isRepeating) {
-
- // All must be selected otherwise size would be zero. Repeating property will not change.
- if (!nullPos[0]) {
- if (!(inputColVector.compareTo(value, 0) <OperatorSymbol> 0)) {
-
- // Entire batch is filtered out.
- batch.size = 0;
- }
- } else {
- batch.size = 0;
- }
- } else if (batch.selectedInUse) {
- int newSize = 0;
- for(int j = 0; j != n; j++) {
- int i = sel[j];
- if (!nullPos[i]) {
- if (inputColVector.compareTo(value, i) <OperatorSymbol> 0) {
- sel[newSize++] = i;
- }
- }
- }
-
- // Change the selected vector
- batch.size = newSize;
- } else {
- int newSize = 0;
- for(int i = 0; i != n; i++) {
- if (!nullPos[i]) {
- if (inputColVector.compareTo(value, i) <OperatorSymbol> 0) {
- sel[newSize++] = i;
- }
- }
- }
- if (newSize < n) {
- batch.size = newSize;
- batch.selectedInUse = true;
- }
- }
- }
- }
-
- @Override
- public int getOutputColumn() {
- return -1;
- }
-
- @Override
- public String getOutputType() {
- return "boolean";
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/IntervalDayTimeColumnCompareIntervalDayTimeColumn.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/IntervalDayTimeColumnCompareIntervalDayTimeColumn.txt b/ql/src/gen/vectorization/ExpressionTemplates/IntervalDayTimeColumnCompareIntervalDayTimeColumn.txt
deleted file mode 100644
index bf62b78..0000000
--- a/ql/src/gen/vectorization/ExpressionTemplates/IntervalDayTimeColumnCompareIntervalDayTimeColumn.txt
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
-
-import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
-
-
-/**
- * Generated from template IntervalDayTimeColumnCompareColumn.txt, which covers comparison
- * expressions between a datetime/interval column and a scalar of the same type. The boolean output
- * is stored in a separate boolean column.
- */
-public class <ClassName> extends <BaseClassName> {
-
- private static final long serialVersionUID = 1L;
-
- public <ClassName>(int colNum1, int colNum2, int outputColumn) {
- super(colNum1, colNum2, outputColumn);
- }
-
- public <ClassName>() {
- super();
- }
-
- @Override
- public VectorExpressionDescriptor.Descriptor getDescriptor() {
- return (new VectorExpressionDescriptor.Builder())
- .setMode(
- VectorExpressionDescriptor.Mode.PROJECTION)
- .setNumArguments(2)
- .setArgumentTypes(
- VectorExpressionDescriptor.ArgumentType.getType("interval_day_time"),
- VectorExpressionDescriptor.ArgumentType.getType("interval_day_time"))
- .setInputExpressionTypes(
- VectorExpressionDescriptor.InputExpressionType.COLUMN,
- VectorExpressionDescriptor.InputExpressionType.COLUMN).build();
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/IntervalDayTimeColumnCompareIntervalDayTimeScalar.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/IntervalDayTimeColumnCompareIntervalDayTimeScalar.txt b/ql/src/gen/vectorization/ExpressionTemplates/IntervalDayTimeColumnCompareIntervalDayTimeScalar.txt
deleted file mode 100644
index 1abb4a3..0000000
--- a/ql/src/gen/vectorization/ExpressionTemplates/IntervalDayTimeColumnCompareIntervalDayTimeScalar.txt
+++ /dev/null
@@ -1,57 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
-
-import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
-
-import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
-
-
-/**
- * Generated from template IntervalDayTimeColumnCompareScalar.txt, which covers comparison
- * expressions between a datetime/interval column and a scalar of the same type. The boolean output
- * is stored in a separate boolean column.
- */
-public class <ClassName> extends <BaseClassName> {
-
- private static final long serialVersionUID = 1L;
-
- public <ClassName>(int colNum, HiveIntervalDayTime value, int outputColumn) {
- super(colNum, value.pisaTimestampUpdate(new PisaTimestamp()), outputColumn);
- }
-
- public <ClassName>() {
- super();
- }
-
- @Override
- public VectorExpressionDescriptor.Descriptor getDescriptor() {
- return (new VectorExpressionDescriptor.Builder())
- .setMode(
- VectorExpressionDescriptor.Mode.PROJECTION)
- .setNumArguments(2)
- .setArgumentTypes(
- VectorExpressionDescriptor.ArgumentType.getType("interval_day_time"),
- VectorExpressionDescriptor.ArgumentType.getType("interval_day_time"))
- .setInputExpressionTypes(
- VectorExpressionDescriptor.InputExpressionType.COLUMN,
- VectorExpressionDescriptor.InputExpressionType.SCALAR).build();
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/IntervalDayTimeScalarCompareIntervalDayTimeColumn.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/IntervalDayTimeScalarCompareIntervalDayTimeColumn.txt b/ql/src/gen/vectorization/ExpressionTemplates/IntervalDayTimeScalarCompareIntervalDayTimeColumn.txt
deleted file mode 100644
index 26762ff..0000000
--- a/ql/src/gen/vectorization/ExpressionTemplates/IntervalDayTimeScalarCompareIntervalDayTimeColumn.txt
+++ /dev/null
@@ -1,57 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
-
-import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
-
-import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
-
-
-/**
- * Generated from template IntervalDayTimeColumnCompareScalar.txt, which covers comparison
- * expressions between a datetime/interval column and a scalar of the same type. The boolean output
- * is stored in a separate boolean column.
- */
-public class <ClassName> extends <BaseClassName> {
-
- private static final long serialVersionUID = 1L;
-
- public <ClassName>(HiveIntervalDayTime value, int colNum, int outputColumn) {
- super(value.pisaTimestampUpdate(new PisaTimestamp()), colNum, outputColumn);
- }
-
- public <ClassName>() {
- super();
- }
-
- @Override
- public VectorExpressionDescriptor.Descriptor getDescriptor() {
- return (new VectorExpressionDescriptor.Builder())
- .setMode(
- VectorExpressionDescriptor.Mode.PROJECTION)
- .setNumArguments(2)
- .setArgumentTypes(
- VectorExpressionDescriptor.ArgumentType.getType("interval_day_time"),
- VectorExpressionDescriptor.ArgumentType.getType("interval_day_time"))
- .setInputExpressionTypes(
- VectorExpressionDescriptor.InputExpressionType.SCALAR,
- VectorExpressionDescriptor.InputExpressionType.COLUMN).build();
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthColumnArithmeticDateColumn.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthColumnArithmeticDateColumn.txt b/ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthColumnArithmeticDateColumn.txt
index 7ae84b7..8e3a419 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthColumnArithmeticDateColumn.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthColumnArithmeticDateColumn.txt
@@ -18,12 +18,15 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
+import java.sql.Date;
+import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil;
import org.apache.hadoop.hive.ql.exec.vector.*;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
import org.apache.hadoop.hive.ql.util.DateTimeMath;
+import org.apache.hadoop.hive.serde2.io.DateWritable;
/**
* Generated from template DateColumnArithmeticIntervalYearMonthColumn.txt, which covers binary arithmetic
@@ -36,12 +39,18 @@ public class <ClassName> extends VectorExpression {
private int colNum1;
private int colNum2;
private int outputColumn;
+ private HiveIntervalYearMonth scratchIntervalYearMonth1;
+ private Date scratchDate2;
+ private Date outputDate;
private DateTimeMath dtm = new DateTimeMath();
public <ClassName>(int colNum1, int colNum2, int outputColumn) {
this.colNum1 = colNum1;
this.colNum2 = colNum2;
this.outputColumn = outputColumn;
+ scratchIntervalYearMonth1 = new HiveIntervalYearMonth();
+ scratchDate2 = new Date(0);
+ outputDate = new Date(0);
}
public <ClassName>() {
@@ -54,10 +63,10 @@ public class <ClassName> extends VectorExpression {
super.evaluateChildren(batch);
}
- // Input #1 is type interval_year_month (months).
+ // Input #1 is type interval_year_month.
LongColumnVector inputColVector1 = (LongColumnVector) batch.cols[colNum1];
- // Input #2 is type date (epochDays).
+ // Input #2 is type date.
LongColumnVector inputColVector2 = (LongColumnVector) batch.cols[colNum2];
// Output is type date.
@@ -89,40 +98,64 @@ public class <ClassName> extends VectorExpression {
* conditional checks in the inner loop.
*/
if (inputColVector1.isRepeating && inputColVector2.isRepeating) {
- outputVector[0] = dtm.addMonthsToDays(vector2[0], <OperatorSymbol> (int) vector1[0]);
+ scratchIntervalYearMonth1.set((int) vector1[0]);
+ scratchDate2.setTime(DateWritable.daysToMillis((int) vector2[0]));
+ dtm.<OperatorMethod>(
+ scratchIntervalYearMonth1, scratchDate2, outputDate);
+ outputVector[0] = DateWritable.dateToDays(outputDate);
} else if (inputColVector1.isRepeating) {
- long value1 = vector1[0];
+ scratchIntervalYearMonth1.set((int) vector1[0]);
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputVector[i] = dtm.addMonthsToDays(vector2[i], <OperatorSymbol> (int) value1);
+ scratchDate2.setTime(DateWritable.daysToMillis((int) vector2[i]));
+ dtm.<OperatorMethod>(
+ scratchIntervalYearMonth1, scratchDate2, outputDate);
+ outputVector[i] = DateWritable.dateToDays(outputDate);
}
} else {
for(int i = 0; i != n; i++) {
- outputVector[i] = dtm.addMonthsToDays(vector2[i], <OperatorSymbol> (int) value1);
+ scratchDate2.setTime(DateWritable.daysToMillis((int) vector2[i]));
+ dtm.<OperatorMethod>(
+ scratchIntervalYearMonth1, scratchDate2, outputDate);
+ outputVector[i] = DateWritable.dateToDays(outputDate);
}
}
} else if (inputColVector2.isRepeating) {
- long value2 = vector2[0];
+ scratchDate2.setTime(DateWritable.daysToMillis((int) vector2[0]));
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputVector[i] = dtm.addMonthsToDays(value2, <OperatorSymbol> (int) vector1[i]);
+ scratchIntervalYearMonth1.set((int) vector1[i]);
+ dtm.<OperatorMethod>(
+ scratchIntervalYearMonth1, scratchDate2, outputDate);
+ outputVector[i] = DateWritable.dateToDays(outputDate);
}
} else {
for(int i = 0; i != n; i++) {
- outputVector[i] = dtm.addMonthsToDays(value2, <OperatorSymbol> (int) vector1[i]);
+ scratchIntervalYearMonth1.set((int) vector1[i]);
+ dtm.<OperatorMethod>(
+ scratchIntervalYearMonth1, scratchDate2, outputDate);
+ outputVector[i] = DateWritable.dateToDays(outputDate);
}
}
} else {
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputVector[i] = dtm.addMonthsToDays(vector2[i], <OperatorSymbol> (int) vector1[i]);
+ scratchIntervalYearMonth1.set((int) vector1[i]);
+ scratchDate2.setTime(DateWritable.daysToMillis((int) vector2[i]));
+ dtm.<OperatorMethod>(
+ scratchIntervalYearMonth1, scratchDate2, outputDate);
+ outputVector[i] = DateWritable.dateToDays(outputDate);
}
} else {
for(int i = 0; i != n; i++) {
- outputVector[i] = dtm.addMonthsToDays(vector2[i], <OperatorSymbol> (int) vector1[i]);
+ scratchIntervalYearMonth1.set((int) vector1[i]);
+ scratchDate2.setTime(DateWritable.daysToMillis((int) vector2[i]));
+ dtm.<OperatorMethod>(
+ scratchIntervalYearMonth1, scratchDate2, outputDate);
+ outputVector[i] = DateWritable.dateToDays(outputDate);
}
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthColumnArithmeticDateScalar.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthColumnArithmeticDateScalar.txt b/ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthColumnArithmeticDateScalar.txt
index 2f2522d..ad65d52 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthColumnArithmeticDateScalar.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthColumnArithmeticDateScalar.txt
@@ -18,6 +18,8 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
+import java.sql.Date;
+import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
@@ -25,6 +27,7 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
import org.apache.hadoop.hive.ql.exec.vector.*;
import org.apache.hadoop.hive.ql.util.DateTimeMath;
+import org.apache.hadoop.hive.serde2.io.DateWritable;
/**
* Generated from template DateColumnArithmeticIntervalYearMonthScalar.txt, which covers binary arithmetic
@@ -35,14 +38,18 @@ public class <ClassName> extends VectorExpression {
private static final long serialVersionUID = 1L;
private int colNum;
- private long value;
+ private Date value;
private int outputColumn;
+ private HiveIntervalYearMonth scratchIntervalYearMonth1;
+ private Date outputDate;
private DateTimeMath dtm = new DateTimeMath();
public <ClassName>(int colNum, long value, int outputColumn) {
this.colNum = colNum;
- this.value = value;
+ this.value = new Date(DateWritable.daysToMillis((int) value));
this.outputColumn = outputColumn;
+ scratchIntervalYearMonth1 = new HiveIntervalYearMonth();
+ outputDate = new Date(0);
}
public <ClassName>() {
@@ -56,18 +63,18 @@ public class <ClassName> extends VectorExpression {
}
// Input #1 is type interval_year_mont (epochMonths).
- LongColumnVector inputColVector = (LongColumnVector) batch.cols[colNum];
+ LongColumnVector inputColVector1 = (LongColumnVector) batch.cols[colNum];
// Output is type date.
LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumn];
int[] sel = batch.selected;
- boolean[] inputIsNull = inputColVector.isNull;
+ boolean[] inputIsNull = inputColVector1.isNull;
boolean[] outputIsNull = outputColVector.isNull;
- outputColVector.noNulls = inputColVector.noNulls;
- outputColVector.isRepeating = inputColVector.isRepeating;
+ outputColVector.noNulls = inputColVector1.noNulls;
+ outputColVector.isRepeating = inputColVector1.isRepeating;
int n = batch.size;
- long[] vector = inputColVector.vector;
+ long[] vector1 = inputColVector1.vector;
long[] outputVector = outputColVector.vector;
// return immediately if batch is empty
@@ -75,32 +82,46 @@ public class <ClassName> extends VectorExpression {
return;
}
- if (inputColVector.isRepeating) {
- outputVector[0] = dtm.addMonthsToDays(value, <OperatorSymbol> (int) vector[0]);
-
+ if (inputColVector1.isRepeating) {
+ scratchIntervalYearMonth1.set((int) vector1[0]);
+ dtm.<OperatorMethod>(
+ scratchIntervalYearMonth1, value, outputDate);
+ outputVector[0] = DateWritable.dateToDays(outputDate);
// Even if there are no nulls, we always copy over entry 0. Simplifies code.
outputIsNull[0] = inputIsNull[0];
- } else if (inputColVector.noNulls) {
+ } else if (inputColVector1.noNulls) {
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputVector[i] = dtm.addMonthsToDays(value, <OperatorSymbol> (int) vector[i]);
+ scratchIntervalYearMonth1.set((int) vector1[i]);
+ dtm.<OperatorMethod>(
+ scratchIntervalYearMonth1, value, outputDate);
+ outputVector[i] = DateWritable.dateToDays(outputDate);
}
} else {
for(int i = 0; i != n; i++) {
- outputVector[i] = dtm.addMonthsToDays(value, <OperatorSymbol> (int) vector[i]);
+ scratchIntervalYearMonth1.set((int) vector1[i]);
+ dtm.<OperatorMethod>(
+ scratchIntervalYearMonth1, value, outputDate);
+ outputVector[i] = DateWritable.dateToDays(outputDate);
}
}
} else /* there are nulls */ {
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputVector[i] = dtm.addMonthsToDays(value, <OperatorSymbol> (int) vector[i]);
+ scratchIntervalYearMonth1.set((int) vector1[i]);
+ dtm.<OperatorMethod>(
+ scratchIntervalYearMonth1, value, outputDate);
+ outputVector[i] = DateWritable.dateToDays(outputDate);
outputIsNull[i] = inputIsNull[i];
}
} else {
for(int i = 0; i != n; i++) {
- outputVector[i] = dtm.addMonthsToDays(value, <OperatorSymbol> (int) vector[i]);
+ scratchIntervalYearMonth1.set((int) vector1[i]);
+ dtm.<OperatorMethod>(
+ scratchIntervalYearMonth1, value, outputDate);
+ outputVector[i] = DateWritable.dateToDays(outputDate);
}
System.arraycopy(inputIsNull, 0, outputIsNull, 0, n);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthColumnArithmeticTimestampColumn.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthColumnArithmeticTimestampColumn.txt b/ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthColumnArithmeticTimestampColumn.txt
index b3da89f..858c3d7 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthColumnArithmeticTimestampColumn.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthColumnArithmeticTimestampColumn.txt
@@ -18,7 +18,9 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
+import java.sql.Timestamp;
+
+import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil;
import org.apache.hadoop.hive.ql.exec.vector.*;
@@ -37,14 +39,14 @@ public class <ClassName> extends VectorExpression {
private int colNum1;
private int colNum2;
private int outputColumn;
- private PisaTimestamp scratchPisaTimestamp;
+ private HiveIntervalYearMonth scratchIntervalYearMonth1;
private DateTimeMath dtm = new DateTimeMath();
public <ClassName>(int colNum1, int colNum2, int outputColumn) {
this.colNum1 = colNum1;
this.colNum2 = colNum2;
this.outputColumn = outputColumn;
- scratchPisaTimestamp = new PisaTimestamp();
+ scratchIntervalYearMonth1 = new HiveIntervalYearMonth();
}
public <ClassName>() {
@@ -57,10 +59,10 @@ public class <ClassName> extends VectorExpression {
super.evaluateChildren(batch);
}
- // Input #1 is type Interval_Year_Month (months).
+ // Input #1 is type Interval_Year_Month.
LongColumnVector inputColVector1 = (LongColumnVector) batch.cols[colNum1];
- // Input #2 is type Timestamp (PisaTimestamp).
+ // Input #2 is type Timestamp.
TimestampColumnVector inputColVector2 = (TimestampColumnVector) batch.cols[colNum2];
// Output is type Timestamp.
@@ -91,54 +93,59 @@ public class <ClassName> extends VectorExpression {
* conditional checks in the inner loop.
*/
if (inputColVector1.isRepeating && inputColVector2.isRepeating) {
- outputColVector.set(0,
- dtm.addMonthsToPisaTimestamp(inputColVector2.asScratchPisaTimestamp(0), <OperatorSymbol> (int) vector1[0],
- scratchPisaTimestamp));
+ scratchIntervalYearMonth1.set((int) vector1[0]);
+ dtm.<OperatorMethod>(
+ scratchIntervalYearMonth1, inputColVector2.asScratchTimestamp(0), outputColVector.getScratchTimestamp());
+ outputColVector.setFromScratchTimestamp(0);
} else if (inputColVector1.isRepeating) {
- long value1 = vector1[0];
+ scratchIntervalYearMonth1.set((int) vector1[0]);
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputColVector.set(i,
- dtm.addMonthsToPisaTimestamp(inputColVector2.asScratchPisaTimestamp(i), <OperatorSymbol> (int) value1,
- scratchPisaTimestamp));
+ dtm.<OperatorMethod>(
+ scratchIntervalYearMonth1, inputColVector2.asScratchTimestamp(i), outputColVector.getScratchTimestamp());
+ outputColVector.setFromScratchTimestamp(i);
}
} else {
for(int i = 0; i != n; i++) {
- outputColVector.set(i,
- dtm.addMonthsToPisaTimestamp(inputColVector2.asScratchPisaTimestamp(i), <OperatorSymbol> (int) value1,
- scratchPisaTimestamp));
+ dtm.<OperatorMethod>(
+ scratchIntervalYearMonth1, inputColVector2.asScratchTimestamp(i), outputColVector.getScratchTimestamp());
+ outputColVector.setFromScratchTimestamp(i);
}
}
} else if (inputColVector2.isRepeating) {
- PisaTimestamp value2 = inputColVector2.asScratchPisaTimestamp(0);
+ Timestamp value2 = inputColVector2.asScratchTimestamp(0);
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputColVector.set(i,
- dtm.addMonthsToPisaTimestamp(value2, <OperatorSymbol> (int) vector1[i],
- scratchPisaTimestamp));
+ scratchIntervalYearMonth1.set((int) vector1[i]);
+ dtm.<OperatorMethod>(
+ scratchIntervalYearMonth1, inputColVector2.asScratchTimestamp(i), outputColVector.getScratchTimestamp());
+ outputColVector.setFromScratchTimestamp(i);
}
} else {
for(int i = 0; i != n; i++) {
- outputColVector.set(i,
- dtm.addMonthsToPisaTimestamp(value2, <OperatorSymbol> (int) vector1[i],
- scratchPisaTimestamp));
+ scratchIntervalYearMonth1.set((int) vector1[i]);
+ dtm.<OperatorMethod>(
+ scratchIntervalYearMonth1, inputColVector2.asScratchTimestamp(i), outputColVector.getScratchTimestamp());
+ outputColVector.setFromScratchTimestamp(i);
}
}
} else {
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputColVector.set(i,
- dtm.addMonthsToPisaTimestamp(inputColVector2.asScratchPisaTimestamp(i), <OperatorSymbol> (int) vector1[i],
- scratchPisaTimestamp));
+ scratchIntervalYearMonth1.set((int) vector1[i]);
+ dtm.<OperatorMethod>(
+ scratchIntervalYearMonth1, inputColVector2.asScratchTimestamp(i), outputColVector.getScratchTimestamp());
+ outputColVector.setFromScratchTimestamp(i);
}
} else {
for(int i = 0; i != n; i++) {
- outputColVector.set(i,
- dtm.addMonthsToPisaTimestamp(inputColVector2.asScratchPisaTimestamp(i), <OperatorSymbol> (int) vector1[i],
- scratchPisaTimestamp));
+ scratchIntervalYearMonth1.set((int) vector1[i]);
+ dtm.<OperatorMethod>(
+ scratchIntervalYearMonth1, inputColVector2.asScratchTimestamp(i), outputColVector.getScratchTimestamp());
+ outputColVector.setFromScratchTimestamp(i);
}
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthColumnArithmeticTimestampScalar.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthColumnArithmeticTimestampScalar.txt b/ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthColumnArithmeticTimestampScalar.txt
index 81f2a77..66fffd2 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthColumnArithmeticTimestampScalar.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthColumnArithmeticTimestampScalar.txt
@@ -20,7 +20,7 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
import java.sql.Timestamp;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
+import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
@@ -38,16 +38,16 @@ public class <ClassName> extends VectorExpression {
private static final long serialVersionUID = 1L;
private int colNum;
- private PisaTimestamp value;
+ private Timestamp value;
private int outputColumn;
- private PisaTimestamp scratchPisaTimestamp;
+ private HiveIntervalYearMonth scratchIntervalYearMonth1;
private DateTimeMath dtm = new DateTimeMath();
public <ClassName>(int colNum, Timestamp value, int outputColumn) {
this.colNum = colNum;
- this.value = new PisaTimestamp(value);
+ this.value = value;
this.outputColumn = outputColumn;
- scratchPisaTimestamp = new PisaTimestamp();
+ scratchIntervalYearMonth1 = new HiveIntervalYearMonth();
}
public <ClassName>() {
@@ -60,7 +60,7 @@ public class <ClassName> extends VectorExpression {
super.evaluateChildren(batch);
}
- // Input #1 is type interval_year_month (epochMonths).
+ // Input #1 is type interval_year_month.
LongColumnVector inputColVector1 = (LongColumnVector) batch.cols[colNum];
// Output is type Timestamp.
@@ -81,41 +81,45 @@ public class <ClassName> extends VectorExpression {
}
if (inputColVector1.isRepeating) {
- outputColVector.set(0,
- dtm.addMonthsToPisaTimestamp(value, <OperatorSymbol> (int) vector1[0],
- scratchPisaTimestamp));
-
+ scratchIntervalYearMonth1.set((int) vector1[0]);
+ dtm.<OperatorMethod>(
+ scratchIntervalYearMonth1, value, outputColVector.getScratchTimestamp());
+ outputColVector.setFromScratchTimestamp(0);
// Even if there are no nulls, we always copy over entry 0. Simplifies code.
outputIsNull[0] = inputIsNull[0];
} else if (inputColVector1.noNulls) {
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputColVector.set(i,
- dtm.addMonthsToPisaTimestamp(value, <OperatorSymbol> (int) vector1[i],
- scratchPisaTimestamp));
+ scratchIntervalYearMonth1.set((int) vector1[i]);
+ dtm.<OperatorMethod>(
+ scratchIntervalYearMonth1, value, outputColVector.getScratchTimestamp());
+ outputColVector.setFromScratchTimestamp(i);
}
} else {
for(int i = 0; i != n; i++) {
- outputColVector.set(i,
- dtm.addMonthsToPisaTimestamp(value, <OperatorSymbol> (int) vector1[i],
- scratchPisaTimestamp));
+ scratchIntervalYearMonth1.set((int) vector1[i]);
+ dtm.<OperatorMethod>(
+ scratchIntervalYearMonth1, value, outputColVector.getScratchTimestamp());
+ outputColVector.setFromScratchTimestamp(i);
}
}
} else /* there are nulls */ {
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputColVector.set(i,
- dtm.addMonthsToPisaTimestamp(value, <OperatorSymbol> (int) vector1[i],
- scratchPisaTimestamp));
+ scratchIntervalYearMonth1.set((int) vector1[i]);
+ dtm.<OperatorMethod>(
+ scratchIntervalYearMonth1, value, outputColVector.getScratchTimestamp());
+ outputColVector.setFromScratchTimestamp(i);
outputIsNull[i] = inputIsNull[i];
}
} else {
for(int i = 0; i != n; i++) {
- outputColVector.set(i,
- dtm.addMonthsToPisaTimestamp(value, <OperatorSymbol> (int) vector1[i],
- scratchPisaTimestamp));
+ scratchIntervalYearMonth1.set((int) vector1[i]);
+ dtm.<OperatorMethod>(
+ scratchIntervalYearMonth1, value, outputColVector.getScratchTimestamp());
+ outputColVector.setFromScratchTimestamp(i);
}
System.arraycopy(inputIsNull, 0, outputIsNull, 0, n);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthScalarArithmeticDateColumn.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthScalarArithmeticDateColumn.txt b/ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthScalarArithmeticDateColumn.txt
index 3f4f05f..ddde913 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthScalarArithmeticDateColumn.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthScalarArithmeticDateColumn.txt
@@ -18,6 +18,8 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
+import java.sql.Date;
+import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
import org.apache.hadoop.hive.ql.exec.vector.*;
@@ -33,6 +35,7 @@ import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil;
import org.apache.hadoop.hive.ql.util.DateTimeMath;
+import org.apache.hadoop.hive.serde2.io.DateWritable;
/**
* Generated from template DateTimeScalarArithmeticIntervalYearMonthColumn.txt.
@@ -44,14 +47,18 @@ public class <ClassName> extends VectorExpression {
private static final long serialVersionUID = 1L;
private int colNum;
- private long value;
+ private HiveIntervalYearMonth value;
private int outputColumn;
+ private Date scratchDate2;
+ private Date outputDate;
private DateTimeMath dtm = new DateTimeMath();
public <ClassName>(long value, int colNum, int outputColumn) {
this.colNum = colNum;
- this.value = value;
+ this.value = new HiveIntervalYearMonth((int) value);
this.outputColumn = outputColumn;
+ scratchDate2 = new Date(0);
+ outputDate = new Date(0);
}
public <ClassName>() {
@@ -70,18 +77,18 @@ public class <ClassName> extends VectorExpression {
}
// Input #2 is type date.
- LongColumnVector inputColVector = (LongColumnVector) batch.cols[colNum];
+ LongColumnVector inputColVector2 = (LongColumnVector) batch.cols[colNum];
// Output is type Date.
LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumn];
int[] sel = batch.selected;
- boolean[] inputIsNull = inputColVector.isNull;
+ boolean[] inputIsNull = inputColVector2.isNull;
boolean[] outputIsNull = outputColVector.isNull;
- outputColVector.noNulls = inputColVector.noNulls;
- outputColVector.isRepeating = inputColVector.isRepeating;
+ outputColVector.noNulls = inputColVector2.noNulls;
+ outputColVector.isRepeating = inputColVector2.isRepeating;
int n = batch.size;
- long[] vector = inputColVector.vector;
+ long[] vector2 = inputColVector2.vector;
long[] outputVector = outputColVector.vector;
// return immediately if batch is empty
@@ -89,32 +96,46 @@ public class <ClassName> extends VectorExpression {
return;
}
- if (inputColVector.isRepeating) {
- outputVector[0] = dtm.addMonthsToDays(vector[0], <OperatorSymbol> (int) value);
-
+ if (inputColVector2.isRepeating) {
+ scratchDate2.setTime(DateWritable.daysToMillis((int) vector2[0]));
+ dtm.<OperatorMethod>(
+ value, scratchDate2, outputDate);
+ outputVector[0] = DateWritable.dateToDays(outputDate);
// Even if there are no nulls, we always copy over entry 0. Simplifies code.
outputIsNull[0] = inputIsNull[0];
- } else if (inputColVector.noNulls) {
+ } else if (inputColVector2.noNulls) {
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputVector[i] = dtm.addMonthsToDays(vector[i], <OperatorSymbol> (int) value);
+ scratchDate2.setTime(DateWritable.daysToMillis((int) vector2[i]));
+ dtm.<OperatorMethod>(
+ value, scratchDate2, outputDate);
+ outputVector[i] = DateWritable.dateToDays(outputDate);
}
} else {
for(int i = 0; i != n; i++) {
- outputVector[i] = dtm.addMonthsToDays(vector[i], <OperatorSymbol> (int) value);
+ scratchDate2.setTime(DateWritable.daysToMillis((int) vector2[i]));
+ dtm.<OperatorMethod>(
+ value, scratchDate2, outputDate);
+ outputVector[i] = DateWritable.dateToDays(outputDate);
}
}
} else { /* there are nulls */
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputVector[i] = dtm.addMonthsToDays(vector[i], <OperatorSymbol> (int) value);
+ scratchDate2.setTime(DateWritable.daysToMillis((int) vector2[i]));
+ dtm.<OperatorMethod>(
+ value, scratchDate2, outputDate);
+ outputVector[i] = DateWritable.dateToDays(outputDate);
outputIsNull[i] = inputIsNull[i];
}
} else {
for(int i = 0; i != n; i++) {
- outputVector[i] = dtm.addMonthsToDays(vector[i], <OperatorSymbol> (int) value);
+ scratchDate2.setTime(DateWritable.daysToMillis((int) vector2[i]));
+ dtm.<OperatorMethod>(
+ value, scratchDate2, outputDate);
+ outputVector[i] = DateWritable.dateToDays(outputDate);
}
System.arraycopy(inputIsNull, 0, outputIsNull, 0, n);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthScalarArithmeticTimestampColumn.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthScalarArithmeticTimestampColumn.txt b/ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthScalarArithmeticTimestampColumn.txt
index 47d611e..cbb7021 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthScalarArithmeticTimestampColumn.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthScalarArithmeticTimestampColumn.txt
@@ -18,11 +18,13 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
+import java.sql.Timestamp;
+
+import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
import org.apache.hadoop.hive.ql.exec.vector.*;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
/*
* Because of the templatized nature of the code, either or both
* of these ColumnVector imports may be needed. Listing both of them
@@ -44,16 +46,14 @@ public class <ClassName> extends VectorExpression {
private static final long serialVersionUID = 1L;
private int colNum;
- private long value;
+ private HiveIntervalYearMonth value;
private int outputColumn;
- private PisaTimestamp scratchPisaTimestamp;
private DateTimeMath dtm = new DateTimeMath();
public <ClassName>(long value, int colNum, int outputColumn) {
this.colNum = colNum;
- this.value = value;
+ this.value = new HiveIntervalYearMonth((int) value);
this.outputColumn = outputColumn;
- scratchPisaTimestamp = new PisaTimestamp();
}
public <ClassName>() {
@@ -72,16 +72,16 @@ public class <ClassName> extends VectorExpression {
}
// Input #2 is type timestamp.
- TimestampColumnVector inputColVector = (TimestampColumnVector) batch.cols[colNum];
+ TimestampColumnVector inputColVector2 = (TimestampColumnVector) batch.cols[colNum];
- // Output is type Timestamp.
+ // Output is type Timestamp.
TimestampColumnVector outputColVector = (TimestampColumnVector) batch.cols[outputColumn];
int[] sel = batch.selected;
- boolean[] inputIsNull = inputColVector.isNull;
+ boolean[] inputIsNull = inputColVector2.isNull;
boolean[] outputIsNull = outputColVector.isNull;
- outputColVector.noNulls = inputColVector.noNulls;
- outputColVector.isRepeating = inputColVector.isRepeating;
+ outputColVector.noNulls = inputColVector2.noNulls;
+ outputColVector.isRepeating = inputColVector2.isRepeating;
int n = batch.size;
// return immediately if batch is empty
@@ -89,42 +89,41 @@ public class <ClassName> extends VectorExpression {
return;
}
- if (inputColVector.isRepeating) {
- outputColVector.set(0,
- dtm.addMonthsToPisaTimestamp(inputColVector.asScratchPisaTimestamp(0), <OperatorSymbol> (int) value,
- scratchPisaTimestamp));
-
+ if (inputColVector2.isRepeating) {
+ dtm.<OperatorMethod>(
+ value, inputColVector2.asScratchTimestamp(0), outputColVector.getScratchTimestamp());
+ outputColVector.setFromScratchTimestamp(0);
// Even if there are no nulls, we always copy over entry 0. Simplifies code.
outputIsNull[0] = inputIsNull[0];
- } else if (inputColVector.noNulls) {
+ } else if (inputColVector2.noNulls) {
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputColVector.set(i,
- dtm.addMonthsToPisaTimestamp(inputColVector.asScratchPisaTimestamp(i), <OperatorSymbol> (int) value,
- scratchPisaTimestamp));
+ dtm.<OperatorMethod>(
+ value, inputColVector2.asScratchTimestamp(i), outputColVector.getScratchTimestamp());
+ outputColVector.setFromScratchTimestamp(i);
}
} else {
for(int i = 0; i != n; i++) {
- outputColVector.set(i,
- dtm.addMonthsToPisaTimestamp(inputColVector.asScratchPisaTimestamp(i), <OperatorSymbol> (int) value,
- scratchPisaTimestamp));
+ dtm.<OperatorMethod>(
+ value, inputColVector2.asScratchTimestamp(i), outputColVector.getScratchTimestamp());
+ outputColVector.setFromScratchTimestamp(i);
}
}
} else { /* there are nulls */
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputColVector.set(i,
- dtm.addMonthsToPisaTimestamp(inputColVector.asScratchPisaTimestamp(i), <OperatorSymbol> (int) value,
- scratchPisaTimestamp));
+ dtm.<OperatorMethod>(
+ value, inputColVector2.asScratchTimestamp(i), outputColVector.getScratchTimestamp());
+ outputColVector.setFromScratchTimestamp(i);
outputIsNull[i] = inputIsNull[i];
}
} else {
for(int i = 0; i != n; i++) {
- outputColVector.set(i,
- dtm.addMonthsToPisaTimestamp(inputColVector.asScratchPisaTimestamp(i), <OperatorSymbol> (int) value,
- scratchPisaTimestamp));
+ dtm.<OperatorMethod>(
+ value, inputColVector2.asScratchTimestamp(i), outputColVector.getScratchTimestamp());
+ outputColVector.setFromScratchTimestamp(i);
}
System.arraycopy(inputIsNull, 0, outputIsNull, 0, n);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/LongDoubleColumnCompareTimestampColumn.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/LongDoubleColumnCompareTimestampColumn.txt b/ql/src/gen/vectorization/ExpressionTemplates/LongDoubleColumnCompareTimestampColumn.txt
index e804e2a..9ccfaac 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/LongDoubleColumnCompareTimestampColumn.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/LongDoubleColumnCompareTimestampColumn.txt
@@ -19,7 +19,6 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
import java.sql.Timestamp;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil;
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/LongDoubleColumnCompareTimestampScalar.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/LongDoubleColumnCompareTimestampScalar.txt b/ql/src/gen/vectorization/ExpressionTemplates/LongDoubleColumnCompareTimestampScalar.txt
index 90720ba..c7d8c65 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/LongDoubleColumnCompareTimestampScalar.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/LongDoubleColumnCompareTimestampScalar.txt
@@ -19,7 +19,6 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
import java.sql.Timestamp;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.*;
@@ -41,7 +40,7 @@ public class <ClassName> extends VectorExpression {
public <ClassName>(int colNum, Timestamp value, int outputColumn) {
this.colNum = colNum;
- this.value = new PisaTimestamp(value).<GetTimestampLongDoubleMethod>();
+ this.value = TimestampColumnVector.<GetTimestampLongDoubleMethod>(value);
this.outputColumn = outputColumn;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/LongDoubleScalarCompareTimestampColumn.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/LongDoubleScalarCompareTimestampColumn.txt b/ql/src/gen/vectorization/ExpressionTemplates/LongDoubleScalarCompareTimestampColumn.txt
index 7d3856a..d47bc10 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/LongDoubleScalarCompareTimestampColumn.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/LongDoubleScalarCompareTimestampColumn.txt
@@ -21,6 +21,7 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.*;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticDateColumn.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticDateColumn.txt b/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticDateColumn.txt
index b086a88..27e083d 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticDateColumn.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticDateColumn.txt
@@ -18,28 +18,156 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
+import java.sql.Timestamp;
+
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil;
import org.apache.hadoop.hive.ql.exec.vector.*;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
import org.apache.hadoop.hive.ql.util.DateTimeMath;
+import org.apache.hadoop.hive.serde2.io.DateWritable;
/**
* Generated from template TimestampColumnArithmeticDateColumn.txt, which covers binary arithmetic
* expressions between columns.
*/
-public class <ClassName> extends <BaseClassName> {
+public class <ClassName> extends VectorExpression {
private static final long serialVersionUID = 1L;
+ private int colNum1;
+ private int colNum2;
+ private int outputColumn;
+ private Timestamp scratchTimestamp2;
+ private DateTimeMath dtm = new DateTimeMath();
+
public <ClassName>(int colNum1, int colNum2, int outputColumn) {
- super(colNum1, colNum2, outputColumn);
+ this.colNum1 = colNum1;
+ this.colNum2 = colNum2;
+ this.outputColumn = outputColumn;
+ scratchTimestamp2 = new Timestamp(0);
}
public <ClassName>() {
- super();
+ }
+
+ @Override
+ public void evaluate(VectorizedRowBatch batch) {
+
+ if (childExpressions != null) {
+ super.evaluateChildren(batch);
+ }
+
+ // Input #1 is type <OperandType1>.
+ <InputColumnVectorType1> inputColVector1 = (<InputColumnVectorType1>) batch.cols[colNum1];
+
+ // Input #2 is type date (days). For the math we convert it to a timestamp.
+ LongColumnVector inputColVector2 = (LongColumnVector) batch.cols[colNum2];
+
+ // Output is type <ReturnType>.
+ <OutputColumnVectorType> outputColVector = (<OutputColumnVectorType>) batch.cols[outputColumn];
+
+ int[] sel = batch.selected;
+ int n = batch.size;
+
+ long[] vector2 = inputColVector2.vector;
+
+ // return immediately if batch is empty
+ if (n == 0) {
+ return;
+ }
+
+ outputColVector.isRepeating =
+ inputColVector1.isRepeating && inputColVector2.isRepeating
+ || inputColVector1.isRepeating && !inputColVector1.noNulls && inputColVector1.isNull[0]
+ || inputColVector2.isRepeating && !inputColVector2.noNulls && inputColVector2.isNull[0];
+
+ // Handle nulls first
+ NullUtil.propagateNullsColCol(
+ inputColVector1, inputColVector2, outputColVector, sel, n, batch.selectedInUse);
+
+ /* Disregard nulls for processing. In other words,
+ * the arithmetic operation is performed even if one or
+ * more inputs are null. This is to improve speed by avoiding
+ * conditional checks in the inner loop.
+ */
+ if (inputColVector1.isRepeating && inputColVector2.isRepeating) {
+ scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[0]));
+ dtm.<OperatorMethod>(
+ inputColVector1.asScratch<CamelOperandType1>(0), scratchTimestamp2, outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(0);
+ } else if (inputColVector1.isRepeating) {
+ <HiveOperandType1> value1 = inputColVector1.asScratch<CamelOperandType1>(0);
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[i]));
+ dtm.<OperatorMethod>(
+ value1, scratchTimestamp2, outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[i]));
+ dtm.<OperatorMethod>(
+ value1, scratchTimestamp2, outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ }
+ }
+ } else if (inputColVector2.isRepeating) {
+ scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[0]));
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ dtm.<OperatorMethod>(
+ inputColVector1.asScratch<CamelOperandType1>(i), scratchTimestamp2, outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ dtm.<OperatorMethod>(
+ inputColVector1.asScratch<CamelOperandType1>(i), scratchTimestamp2, outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ }
+ }
+ } else {
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[i]));
+ dtm.<OperatorMethod>(
+ inputColVector1.asScratch<CamelOperandType1>(i), scratchTimestamp2, outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[i]));
+ dtm.<OperatorMethod>(
+ inputColVector1.asScratch<CamelOperandType1>(i), scratchTimestamp2, outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ }
+ }
+ }
+
+ /* For the case when the output can have null values, follow
+ * the convention that the data values must be 1 for long and
+ * NaN for double. This is to prevent possible later zero-divide errors
+ * in complex arithmetic expressions like col2 / (col1 - 1)
+ * in the case when some col1 entries are null.
+ */
+ NullUtil.setNullDataEntries<CamelReturnType>(outputColVector, batch.selectedInUse, sel, n);
+ }
+
+ @Override
+ public int getOutputColumn() {
+ return outputColumn;
+ }
+
+ @Override
+ public String getOutputType() {
+ return "interval_day_time";
}
@Override
@@ -50,7 +178,7 @@ public class <ClassName> extends <BaseClassName> {
.setNumArguments(2)
.setArgumentTypes(
VectorExpressionDescriptor.ArgumentType.getType("<OperandType1>"),
- VectorExpressionDescriptor.ArgumentType.getType("<OperandType2>"))
+ VectorExpressionDescriptor.ArgumentType.getType("date"))
.setInputExpressionTypes(
VectorExpressionDescriptor.InputExpressionType.COLUMN,
VectorExpressionDescriptor.InputExpressionType.COLUMN).build();
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticDateColumnBase.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticDateColumnBase.txt b/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticDateColumnBase.txt
deleted file mode 100644
index 7f5496c..0000000
--- a/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticDateColumnBase.txt
+++ /dev/null
@@ -1,172 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
-
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil;
-import org.apache.hadoop.hive.ql.exec.vector.*;
-import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
-import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
-import org.apache.hadoop.hive.serde2.io.DateWritable;
-
-/**
- * Generated from template TimestampColumnArithmeticDateColumnBase.txt, which covers binary arithmetic
- * expressions between columns.
- */
-public abstract class <BaseClassName> extends VectorExpression {
-
- private static final long serialVersionUID = 1L;
-
- private int colNum1;
- private int colNum2;
- private int outputColumn;
- private PisaTimestamp scratchPisaTimestamp;
-
- public <BaseClassName>(int colNum1, int colNum2, int outputColumn) {
- this.colNum1 = colNum1;
- this.colNum2 = colNum2;
- this.outputColumn = outputColumn;
- scratchPisaTimestamp = new PisaTimestamp();
- }
-
- public <BaseClassName>() {
- }
-
- @Override
- public void evaluate(VectorizedRowBatch batch) {
-
- if (childExpressions != null) {
- super.evaluateChildren(batch);
- }
-
- // Input #1 is type timestamp (PisaTimestamp).
- TimestampColumnVector inputColVector1 = (TimestampColumnVector) batch.cols[colNum1];
-
- // Input #2 is type date.
- LongColumnVector inputColVector2 = (LongColumnVector) batch.cols[colNum2];
-
- // Output is type timestamp.
- TimestampColumnVector outputColVector = (TimestampColumnVector) batch.cols[outputColumn];
-
- int[] sel = batch.selected;
- int n = batch.size;
-
- long[] vector2 = inputColVector2.vector;
-
- // return immediately if batch is empty
- if (n == 0) {
- return;
- }
-
- outputColVector.isRepeating =
- inputColVector1.isRepeating && inputColVector2.isRepeating
- || inputColVector1.isRepeating && !inputColVector1.noNulls && inputColVector1.isNull[0]
- || inputColVector2.isRepeating && !inputColVector2.noNulls && inputColVector2.isNull[0];
-
- // Handle nulls first
- NullUtil.propagateNullsColCol(
- inputColVector1, inputColVector2, outputColVector, sel, n, batch.selectedInUse);
-
- /* Disregard nulls for processing. In other words,
- * the arithmetic operation is performed even if one or
- * more inputs are null. This is to improve speed by avoiding
- * conditional checks in the inner loop.
- */
- if (inputColVector1.isRepeating && inputColVector2.isRepeating) {
- outputColVector.<OperatorMethod>(
- inputColVector1.asScratchPisaTimestamp(0),
- scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector2[0])),
- 0);
- } else if (inputColVector1.isRepeating) {
- PisaTimestamp value1 = inputColVector1.asScratchPisaTimestamp(0);
- if (batch.selectedInUse) {
- for(int j = 0; j != n; j++) {
- int i = sel[j];
- outputColVector.<OperatorMethod>(
- value1,
- scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector2[i])),
- i);
- }
- } else {
- for(int i = 0; i != n; i++) {
- outputColVector.<OperatorMethod>(
- value1,
- scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector2[i])),
- i);
- }
- }
- } else if (inputColVector2.isRepeating) {
- PisaTimestamp value2 =
- scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector2[0]));
- if (batch.selectedInUse) {
- for(int j = 0; j != n; j++) {
- int i = sel[j];
- outputColVector.<OperatorMethod>(
- inputColVector1.asScratchPisaTimestamp(i),
- value2,
- i);
- }
- } else {
- for(int i = 0; i != n; i++) {
- outputColVector.<OperatorMethod>(
- inputColVector1.asScratchPisaTimestamp(i),
- value2,
- i);
- }
- }
- } else {
- if (batch.selectedInUse) {
- for(int j = 0; j != n; j++) {
- int i = sel[j];
- outputColVector.<OperatorMethod>(
- inputColVector1.asScratchPisaTimestamp(i),
- scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector2[i])),
- i);
- }
- } else {
- for(int i = 0; i != n; i++) {
- outputColVector.<OperatorMethod>(
- inputColVector1.asScratchPisaTimestamp(i),
- scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector2[i])),
- i);
- }
- }
- }
-
- /* For the case when the output can have null values, follow
- * the convention that the data values must be 1 for long and
- * NaN for double. This is to prevent possible later zero-divide errors
- * in complex arithmetic expressions like col2 / (col1 - 1)
- * in the case when some col1 entries are null.
- */
- NullUtil.setNullDataEntriesTimestamp(outputColVector, batch.selectedInUse, sel, n);
- }
-
- @Override
- public int getOutputColumn() {
- return outputColumn;
- }
-
- @Override
- public String getOutputType() {
- return "timestamp";
- }
-}
-
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticDateScalar.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticDateScalar.txt b/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticDateScalar.txt
index b8404db..8b91a4a 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticDateScalar.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticDateScalar.txt
@@ -18,7 +18,9 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
+import java.sql.Timestamp;
+
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
@@ -26,21 +28,107 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
import org.apache.hadoop.hive.ql.exec.vector.*;
import org.apache.hadoop.hive.ql.util.DateTimeMath;
+import org.apache.hadoop.hive.serde2.io.DateWritable;
/**
* Generated from template TimestampColumnArithmeticDateScalar.txt, which covers binary arithmetic
* expressions between a column and a scalar.
*/
-public class <ClassName> extends <BaseClassName> {
+public class <ClassName> extends VectorExpression {
private static final long serialVersionUID = 1L;
+ private int colNum;
+ private Timestamp value;
+ private int outputColumn;
+ private DateTimeMath dtm = new DateTimeMath();
+
public <ClassName>(int colNum, long value, int outputColumn) {
- super(colNum, value, outputColumn);
+ this.colNum = colNum;
+ this.value = new Timestamp(0);
+ this.value.setTime(DateWritable.daysToMillis((int) value));
+ this.outputColumn = outputColumn;
}
public <ClassName>() {
- super();
+ }
+
+ @Override
+ public void evaluate(VectorizedRowBatch batch) {
+
+ if (childExpressions != null) {
+ super.evaluateChildren(batch);
+ }
+
+ // Input #1 is type <OperandType1>.
+ <InputColumnVectorType1> inputColVector1 = (<InputColumnVectorType1>) batch.cols[colNum];
+
+ // Output is type <ReturnType>.
+ <OutputColumnVectorType> outputColVector = (<OutputColumnVectorType>) batch.cols[outputColumn];
+
+ int[] sel = batch.selected;
+ boolean[] inputIsNull = inputColVector1.isNull;
+ boolean[] outputIsNull = outputColVector.isNull;
+ outputColVector.noNulls = inputColVector1.noNulls;
+ outputColVector.isRepeating = inputColVector1.isRepeating;
+ int n = batch.size;
+
+ // return immediately if batch is empty
+ if (n == 0) {
+ return;
+ }
+
+ if (inputColVector1.isRepeating) {
+ dtm.<OperatorMethod>(
+ inputColVector1.asScratch<CamelOperandType1>(0), value, outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(0);
+ // Even if there are no nulls, we always copy over entry 0. Simplifies code.
+ outputIsNull[0] = inputIsNull[0];
+ } else if (inputColVector1.noNulls) {
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ dtm.<OperatorMethod>(
+ inputColVector1.asScratch<CamelOperandType1>(i), value, outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ dtm.<OperatorMethod>(
+ inputColVector1.asScratch<CamelOperandType1>(i), value, outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ }
+ }
+ } else /* there are nulls */ {
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ dtm.<OperatorMethod>(
+ inputColVector1.asScratch<CamelOperandType1>(i), value, outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ outputIsNull[i] = inputIsNull[i];
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ dtm.<OperatorMethod>(
+ inputColVector1.asScratch<CamelOperandType1>(i), value, outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ }
+ System.arraycopy(inputIsNull, 0, outputIsNull, 0, n);
+ }
+ }
+
+ NullUtil.setNullOutputEntriesColScalar(outputColVector, batch.selectedInUse, sel, n);
+ }
+
+ @Override
+ public int getOutputColumn() {
+ return outputColumn;
+ }
+
+ @Override
+ public String getOutputType() {
+ return "<ReturnType>";
}
@Override
@@ -51,7 +139,7 @@ public class <ClassName> extends <BaseClassName> {
.setNumArguments(2)
.setArgumentTypes(
VectorExpressionDescriptor.ArgumentType.getType("<OperandType1>"),
- VectorExpressionDescriptor.ArgumentType.getType("<OperandType2>"))
+ VectorExpressionDescriptor.ArgumentType.getType("date"))
.setInputExpressionTypes(
VectorExpressionDescriptor.InputExpressionType.COLUMN,
VectorExpressionDescriptor.InputExpressionType.SCALAR).build();
[15/50] [abbrv] hive git commit: HIVE-12552 : Wrong number of reducer
estimation causing job to fail (Rajesh Balamohan via Gunther Hagleitner)
Posted by jd...@apache.org.
HIVE-12552 : Wrong number of reducer estimation causing job to fail (Rajesh Balamohan via Gunther Hagleitner)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b75d9ea8
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b75d9ea8
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b75d9ea8
Branch: refs/heads/llap
Commit: b75d9ea8a73f85d1420f8e3ba1e3b8f9b9acdc5e
Parents: b1c4502
Author: Rajesh Balamohan <rbalamohan at apache dot org>
Authored: Wed Dec 9 11:48:00 2015 -0800
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Fri Mar 25 07:21:55 2016 -0700
----------------------------------------------------------------------
ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b75d9ea8/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
index 8eab3af..d5a2eca 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
@@ -119,7 +119,8 @@ public class GenTezUtils {
// max we allow tez to pick
int maxPartition = (int) (reduceSink.getConf().getNumReducers() * maxPartitionFactor);
- maxPartition = (maxPartition > maxReducers) ? maxReducers : maxPartition;
+ maxPartition = Math.max(1, (maxPartition > maxReducers) ? maxReducers :
+ maxPartition);
reduceWork.setMinReduceTasks(minPartition);
reduceWork.setMaxReduceTasks(maxPartition);
[42/50] [abbrv] hive git commit: HIVE-13326: HiveServer2: Make ZK
config publishing configurable (Vaibhav Gumashta reviewed by Thejas Nair)
Posted by jd...@apache.org.
HIVE-13326: HiveServer2: Make ZK config publishing configurable (Vaibhav Gumashta reviewed by Thejas Nair)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/56b64598
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/56b64598
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/56b64598
Branch: refs/heads/llap
Commit: 56b645981cf466830daaed98f978df5f509bd149
Parents: a14ef8a
Author: Vaibhav Gumashta <vg...@hortonworks.com>
Authored: Tue Mar 29 12:57:47 2016 -0700
Committer: Vaibhav Gumashta <vg...@hortonworks.com>
Committed: Tue Mar 29 12:57:47 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 2 +
.../org/apache/hive/jdbc/miniHS2/MiniHS2.java | 56 +++++++-
.../jdbc/TestJdbcWithLocalClusterSpark.java | 2 +-
.../apache/hive/jdbc/TestJdbcWithMiniMr.java | 2 +-
...stMultiSessionsHS2WithLocalClusterSpark.java | 6 +-
.../jdbc/TestServiceDiscoveryWithMiniHS2.java | 132 +++++++++++++++++++
.../jdbc/authorization/TestHS2AuthzContext.java | 4 +-
.../authorization/TestJdbcMetadataApiAuth.java | 2 +-
.../TestJdbcWithSQLAuthorization.java | 2 +-
.../hive/jdbc/ZooKeeperHiveClientHelper.java | 21 ++-
.../apache/hive/service/server/HiveServer2.java | 39 +++---
11 files changed, 236 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/56b64598/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index f03c1ab..95c5c0e 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2017,6 +2017,8 @@ public class HiveConf extends Configuration {
"hive.zookeeper.quorum in their connection string."),
HIVE_SERVER2_ZOOKEEPER_NAMESPACE("hive.server2.zookeeper.namespace", "hiveserver2",
"The parent node in ZooKeeper used by HiveServer2 when supporting dynamic service discovery."),
+ HIVE_SERVER2_ZOOKEEPER_PUBLISH_CONFIGS("hive.server2.zookeeper.publish.configs", true,
+ "Whether we should publish HiveServer2's configs to ZooKeeper."),
// HiveServer2 global init file location
HIVE_SERVER2_GLOBAL_INIT_FILE_LOCATION("hive.server2.global.init.file.location", "${env:HIVE_CONF_DIR}",
http://git-wip-us.apache.org/repos/asf/hive/blob/56b64598/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java b/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
index 751d8ea..a9d9c76 100644
--- a/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
+++ b/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.ql.WindowsPathUtil;
import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.util.ZooKeeperHiveHelper;
import org.apache.hadoop.hive.shims.HadoopShims.MiniDFSShim;
import org.apache.hadoop.hive.shims.HadoopShims.MiniMrShim;
import org.apache.hadoop.hive.shims.ShimLoader;
@@ -303,6 +304,13 @@ public class MiniHS2 extends AbstractHiveService {
return getServiceClientInternal();
}
+ public HiveConf getServerConf() {
+ if (hiveServer2 != null) {
+ return hiveServer2.getHiveConf();
+ }
+ return null;
+ }
+
public CLIServiceClient getServiceClientInternal() {
for (Service service : hiveServer2.getServices()) {
if (service instanceof ThriftBinaryCLIService) {
@@ -318,8 +326,9 @@ public class MiniHS2 extends AbstractHiveService {
/**
* return connection URL for this server instance
* @return
+ * @throws Exception
*/
- public String getJdbcURL() {
+ public String getJdbcURL() throws Exception {
return getJdbcURL("default");
}
@@ -327,8 +336,9 @@ public class MiniHS2 extends AbstractHiveService {
* return connection URL for this server instance
* @param dbName - DB name to be included in the URL
* @return
+ * @throws Exception
*/
- public String getJdbcURL(String dbName) {
+ public String getJdbcURL(String dbName) throws Exception {
return getJdbcURL(dbName, "");
}
@@ -337,8 +347,9 @@ public class MiniHS2 extends AbstractHiveService {
* @param dbName - DB name to be included in the URL
* @param sessionConfExt - Addional string to be appended to sessionConf part of url
* @return
+ * @throws Exception
*/
- public String getJdbcURL(String dbName, String sessionConfExt) {
+ public String getJdbcURL(String dbName, String sessionConfExt) throws Exception {
return getJdbcURL(dbName, sessionConfExt, "");
}
@@ -348,8 +359,9 @@ public class MiniHS2 extends AbstractHiveService {
* @param sessionConfExt - Addional string to be appended to sessionConf part of url
* @param hiveConfExt - Additional string to be appended to HiveConf part of url (excluding the ?)
* @return
+ * @throws Exception
*/
- public String getJdbcURL(String dbName, String sessionConfExt, String hiveConfExt) {
+ public String getJdbcURL(String dbName, String sessionConfExt, String hiveConfExt) throws Exception {
sessionConfExt = (sessionConfExt == null ? "" : sessionConfExt);
hiveConfExt = (hiveConfExt == null ? "" : hiveConfExt);
String krbConfig = "";
@@ -359,7 +371,17 @@ public class MiniHS2 extends AbstractHiveService {
if (isHttpTransportMode()) {
sessionConfExt = "transportMode=http;httpPath=cliservice;" + sessionConfExt;
}
- String baseJdbcURL = getBaseJdbcURL() + dbName + ";" + krbConfig + ";" + sessionConfExt;
+ String baseJdbcURL;
+ if (isDynamicServiceDiscovery()) {
+ String serviceDiscoveryConfig =
+ "serviceDiscoveryMode=zooKeeper;zooKeeperNamespace="
+ + getServerConf().getVar(HiveConf.ConfVars.HIVE_SERVER2_ZOOKEEPER_NAMESPACE) + ";";
+ baseJdbcURL = getZKBaseJdbcURL() + dbName + ";" + serviceDiscoveryConfig;
+ }
+ else {
+ baseJdbcURL = getBaseJdbcURL() + dbName + ";";
+ }
+ baseJdbcURL = baseJdbcURL + krbConfig + ";" + sessionConfExt;
if (!hiveConfExt.trim().equals("")) {
baseJdbcURL = "?" + hiveConfExt;
}
@@ -379,11 +401,35 @@ public class MiniHS2 extends AbstractHiveService {
}
}
+ /**
+ * Build zk base JDBC URL
+ * @return
+ */
+ private String getZKBaseJdbcURL() throws Exception {
+ HiveConf hiveConf = getServerConf();
+ if (hiveConf != null) {
+ String zkEnsemble = ZooKeeperHiveHelper.getQuorumServers(hiveConf);
+ return "jdbc:hive2://" + zkEnsemble + "/";
+ }
+ throw new Exception("Server's HiveConf is null. Unable to read ZooKeeper configs.");
+ }
+
private boolean isHttpTransportMode() {
String transportMode = getConfProperty(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname);
return transportMode != null && (transportMode.equalsIgnoreCase(HS2_HTTP_MODE));
}
+ private boolean isDynamicServiceDiscovery() throws Exception {
+ HiveConf hiveConf = getServerConf();
+ if (hiveConf == null) {
+ throw new Exception("Server's HiveConf is null. Unable to read ZooKeeper configs.");
+ }
+ if (hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_SUPPORT_DYNAMIC_SERVICE_DISCOVERY)) {
+ return true;
+ }
+ return false;
+ }
+
public static String getJdbcDriverName() {
return driverName;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/56b64598/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithLocalClusterSpark.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithLocalClusterSpark.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithLocalClusterSpark.java
index f649fc2..cabddea 100644
--- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithLocalClusterSpark.java
+++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithLocalClusterSpark.java
@@ -92,7 +92,7 @@ public class TestJdbcWithLocalClusterSpark {
}
// setup DB
- private static void createDb() throws SQLException {
+ private static void createDb() throws Exception {
Connection conn = DriverManager.
getConnection(miniHS2.getJdbcURL(), System.getProperty("user.name"), "bar");
Statement stmt2 = conn.createStatement();
http://git-wip-us.apache.org/repos/asf/hive/blob/56b64598/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniMr.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniMr.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniMr.java
index bcd65a9..637e51a 100644
--- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniMr.java
+++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniMr.java
@@ -83,7 +83,7 @@ public class TestJdbcWithMiniMr {
}
// setup DB
- private static void createDb() throws SQLException {
+ private static void createDb() throws Exception {
Connection conn = DriverManager.
getConnection(miniHS2.getJdbcURL(), System.getProperty("user.name"), "bar");
Statement stmt2 = conn.createStatement();
http://git-wip-us.apache.org/repos/asf/hive/blob/56b64598/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestMultiSessionsHS2WithLocalClusterSpark.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestMultiSessionsHS2WithLocalClusterSpark.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestMultiSessionsHS2WithLocalClusterSpark.java
index 0c3479d..e3f9646 100644
--- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestMultiSessionsHS2WithLocalClusterSpark.java
+++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestMultiSessionsHS2WithLocalClusterSpark.java
@@ -101,7 +101,7 @@ public class TestMultiSessionsHS2WithLocalClusterSpark {
}
// setup DB
- private static void createDb() throws SQLException {
+ private static void createDb() throws Exception {
Connection conn = DriverManager.
getConnection(miniHS2.getJdbcURL(), System.getProperty("user.name"), "bar");
Statement stmt2 = conn.createStatement();
@@ -124,7 +124,7 @@ public class TestMultiSessionsHS2WithLocalClusterSpark {
closeConnection();
}
- private void createConnection() throws SQLException {
+ private void createConnection() throws Exception {
Connection connection = DriverManager.getConnection(miniHS2.getJdbcURL(dbName),
System.getProperty("user.name"), "bar");
Statement statement = connection.createStatement();
@@ -216,7 +216,7 @@ public class TestMultiSessionsHS2WithLocalClusterSpark {
}
private void testKvQuery(String queryStr, String resultVal)
- throws SQLException {
+ throws Exception {
createConnection();
verifyResult(queryStr, resultVal, 2);
closeConnection();
http://git-wip-us.apache.org/repos/asf/hive/blob/56b64598/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestServiceDiscoveryWithMiniHS2.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestServiceDiscoveryWithMiniHS2.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestServiceDiscoveryWithMiniHS2.java
new file mode 100644
index 0000000..907ccb0
--- /dev/null
+++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestServiceDiscoveryWithMiniHS2.java
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.jdbc;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.curator.test.TestingServer;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hive.jdbc.miniHS2.MiniHS2;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestServiceDiscoveryWithMiniHS2 {
+ private static MiniHS2 miniHS2 = null;
+ private static HiveConf hiveConf;
+ private static TestingServer zkServer;
+ private static String zkRootNamespace = "hs2test";
+ private static String dataFileDir;
+ private static Path kvDataFilePath;
+
+ private Connection hs2Conn = null;
+
+ @BeforeClass
+ public static void beforeTest() throws Exception {
+ zkServer = new TestingServer();
+ Class.forName(MiniHS2.getJdbcDriverName());
+ hiveConf = new HiveConf();
+ hiveConf.setBoolVar(ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
+ // Set up zookeeper dynamic service discovery configs
+ enableZKServiceDiscoveryConfigs(hiveConf);
+ dataFileDir = hiveConf.get("test.data.files").replace('\\', '/').replace("c:", "");
+ kvDataFilePath = new Path(dataFileDir, "kv1.txt");
+ }
+
+ @AfterClass
+ public static void afterTest() throws Exception {
+ if (zkServer != null) {
+ zkServer.close();
+ zkServer = null;
+ }
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ miniHS2 = new MiniHS2(hiveConf);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (hs2Conn != null) {
+ hs2Conn.close();
+ }
+ if ((miniHS2 != null) && miniHS2.isStarted()) {
+ miniHS2.stop();
+ }
+ }
+
+ @Test
+ public void testConnectionWithConfigsPublished() throws Exception {
+ Map<String, String> confOverlay = new HashMap<String, String>();
+ confOverlay.put("hive.server2.zookeeper.publish.configs", "true");
+ miniHS2.start(confOverlay);
+ openConnectionAndRunQuery();
+ }
+
+ @Test
+ public void testConnectionWithoutConfigsPublished() throws Exception {
+ Map<String, String> confOverlay = new HashMap<String, String>();
+ confOverlay.put("hive.server2.zookeeper.publish.configs", "false");
+ miniHS2.start(confOverlay);
+ openConnectionAndRunQuery();
+ }
+
+ private static void enableZKServiceDiscoveryConfigs(HiveConf conf) {
+ conf.setBoolVar(ConfVars.HIVE_SERVER2_SUPPORT_DYNAMIC_SERVICE_DISCOVERY, true);
+ conf.setVar(ConfVars.HIVE_ZOOKEEPER_QUORUM, zkServer.getConnectString());
+ conf.setVar(ConfVars.HIVE_SERVER2_ZOOKEEPER_NAMESPACE, zkRootNamespace);
+ }
+
+ private Connection getConnection(String jdbcURL, String user, String pwd) throws SQLException {
+ Connection conn = DriverManager.getConnection(jdbcURL, user, pwd);
+ return conn;
+ }
+
+ private void openConnectionAndRunQuery() throws Exception {
+ hs2Conn = getConnection(miniHS2.getJdbcURL(), System.getProperty("user.name"), "bar");
+ String tableName = "testTab1";
+ Statement stmt = hs2Conn.createStatement();
+ // create table
+ stmt.execute("DROP TABLE IF EXISTS " + tableName);
+ stmt.execute("CREATE TABLE " + tableName
+ + " (under_col INT COMMENT 'the under column', value STRING) COMMENT ' test table'");
+ // load data
+ stmt.execute("load data local inpath '" + kvDataFilePath.toString() + "' into table "
+ + tableName);
+ ResultSet res = stmt.executeQuery("SELECT * FROM " + tableName);
+ assertTrue(res.next());
+ assertEquals("val_238", res.getString(2));
+ res.close();
+ stmt.close();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/56b64598/itests/hive-unit/src/test/java/org/apache/hive/jdbc/authorization/TestHS2AuthzContext.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/authorization/TestHS2AuthzContext.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/authorization/TestHS2AuthzContext.java
index 0bb3c0a..c43776b 100644
--- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/authorization/TestHS2AuthzContext.java
+++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/authorization/TestHS2AuthzContext.java
@@ -101,7 +101,7 @@ public class TestHS2AuthzContext {
verifyContextContents("dfs -ls /", "-ls /");
}
- private void verifyContextContents(final String cmd, String ctxCmd) throws SQLException,
+ private void verifyContextContents(final String cmd, String ctxCmd) throws Exception,
HiveAuthzPluginException, HiveAccessControlException {
Connection hs2Conn = getConnection("user1");
Statement stmt = hs2Conn.createStatement();
@@ -126,7 +126,7 @@ public class TestHS2AuthzContext {
}
- private Connection getConnection(String userName) throws SQLException {
+ private Connection getConnection(String userName) throws Exception {
return DriverManager.getConnection(miniHS2.getJdbcURL(), userName, "bar");
}
http://git-wip-us.apache.org/repos/asf/hive/blob/56b64598/itests/hive-unit/src/test/java/org/apache/hive/jdbc/authorization/TestJdbcMetadataApiAuth.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/authorization/TestJdbcMetadataApiAuth.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/authorization/TestJdbcMetadataApiAuth.java
index 19b311d..692bfa0 100644
--- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/authorization/TestJdbcMetadataApiAuth.java
+++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/authorization/TestJdbcMetadataApiAuth.java
@@ -253,7 +253,7 @@ public class TestJdbcMetadataApiAuth {
}
}
- private static Connection getConnection(String userName) throws SQLException {
+ private static Connection getConnection(String userName) throws Exception {
return DriverManager.getConnection(miniHS2.getJdbcURL(), userName, "bar");
}
http://git-wip-us.apache.org/repos/asf/hive/blob/56b64598/itests/hive-unit/src/test/java/org/apache/hive/jdbc/authorization/TestJdbcWithSQLAuthorization.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/authorization/TestJdbcWithSQLAuthorization.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/authorization/TestJdbcWithSQLAuthorization.java
index dacde45..5e653ec 100644
--- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/authorization/TestJdbcWithSQLAuthorization.java
+++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/authorization/TestJdbcWithSQLAuthorization.java
@@ -126,7 +126,7 @@ public class TestJdbcWithSQLAuthorization {
}
}
- private Connection getConnection(String userName) throws SQLException {
+ private Connection getConnection(String userName) throws Exception {
return DriverManager.getConnection(miniHS2.getJdbcURL(), userName, "bar");
}
http://git-wip-us.apache.org/repos/asf/hive/blob/56b64598/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java
----------------------------------------------------------------------
diff --git a/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java b/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java
index 1ca77a1..8d6003a 100644
--- a/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java
+++ b/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java
@@ -71,12 +71,27 @@ class ZooKeeperHiveClientHelper {
// Now pick a server node randomly
serverNode = serverHosts.get(randomizer.nextInt(serverHosts.size()));
connParams.setCurrentHostZnodePath(serverNode);
- // Read config string from the znode for this server node
- String serverConfStr =
+ // Read data from the znode for this server node
+ // This data could be either config string (new releases) or server end
+ // point (old releases)
+ String dataStr =
new String(
zooKeeperClient.getData().forPath("/" + zooKeeperNamespace + "/" + serverNode),
Charset.forName("UTF-8"));
- applyConfs(serverConfStr, connParams);
+ Matcher matcher = kvPattern.matcher(dataStr);
+ // If dataStr is not null and dataStr is not a KV pattern,
+ // it must be the server uri added by an older version HS2
+ if ((dataStr != null) && (!matcher.find())) {
+ String[] split = dataStr.split(":");
+ if (split.length != 2) {
+ throw new ZooKeeperHiveClientException("Unable to read HiveServer2 uri from ZooKeeper: "
+ + dataStr);
+ }
+ connParams.setHost(split[0]);
+ connParams.setPort(Integer.parseInt(split[1]));
+ } else {
+ applyConfs(dataStr, connParams);
+ }
} catch (Exception e) {
throw new ZooKeeperHiveClientException("Unable to read HiveServer2 configs from ZooKeeper", e);
} finally {
http://git-wip-us.apache.org/repos/asf/hive/blob/56b64598/service/src/java/org/apache/hive/service/server/HiveServer2.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/server/HiveServer2.java b/service/src/java/org/apache/hive/service/server/HiveServer2.java
index ab834b9..d95f78f 100644
--- a/service/src/java/org/apache/hive/service/server/HiveServer2.java
+++ b/service/src/java/org/apache/hive/service/server/HiveServer2.java
@@ -254,10 +254,6 @@ public class HiveServer2 extends CompositeService {
String rootNamespace = hiveConf.getVar(HiveConf.ConfVars.HIVE_SERVER2_ZOOKEEPER_NAMESPACE);
String instanceURI = getServerInstanceURI();
setUpZooKeeperAuth(hiveConf);
- // HiveServer2 configs that this instance will publish to ZooKeeper,
- // so that the clients can read these and configure themselves properly.
- Map<String, String> confsToPublish = new HashMap<String, String>();
- addConfsToPublish(hiveConf, confsToPublish);
int sessionTimeout =
(int) hiveConf.getTimeVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_SESSION_TIMEOUT,
TimeUnit.MILLISECONDS);
@@ -291,8 +287,16 @@ public class HiveServer2 extends CompositeService {
+ ZooKeeperHiveHelper.ZOOKEEPER_PATH_SEPARATOR + "serverUri=" + instanceURI + ";"
+ "version=" + HiveVersionInfo.getVersion() + ";" + "sequence=";
String znodeData = "";
- // Publish configs for this instance as the data on the node
- znodeData = Joiner.on(';').withKeyValueSeparator("=").join(confsToPublish);
+ if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ZOOKEEPER_PUBLISH_CONFIGS)) {
+ // HiveServer2 configs that this instance will publish to ZooKeeper,
+ // so that the clients can read these and configure themselves properly.
+ Map<String, String> confsToPublish = new HashMap<String, String>();
+ addConfsToPublish(hiveConf, confsToPublish);
+ // Publish configs for this instance as the data on the node
+ znodeData = Joiner.on(';').withKeyValueSeparator("=").join(confsToPublish);
+ } else {
+ znodeData = instanceURI;
+ }
byte[] znodeDataUTF8 = znodeData.getBytes(Charset.forName("UTF-8"));
znode =
new PersistentEphemeralNode(zooKeeperClient,
@@ -334,12 +338,12 @@ public class HiveServer2 extends CompositeService {
// Transport specific confs
if (isHTTPTransportMode(hiveConf)) {
confsToPublish.put(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT.varname,
- hiveConf.getVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT));
+ Integer.toString(hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT)));
confsToPublish.put(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PATH.varname,
hiveConf.getVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PATH));
} else {
confsToPublish.put(ConfVars.HIVE_SERVER2_THRIFT_PORT.varname,
- hiveConf.getVar(ConfVars.HIVE_SERVER2_THRIFT_PORT));
+ Integer.toString(hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_PORT)));
confsToPublish.put(ConfVars.HIVE_SERVER2_THRIFT_SASL_QOP.varname,
hiveConf.getVar(ConfVars.HIVE_SERVER2_THRIFT_SASL_QOP));
}
@@ -352,7 +356,7 @@ public class HiveServer2 extends CompositeService {
}
// SSL conf
confsToPublish.put(ConfVars.HIVE_SERVER2_USE_SSL.varname,
- hiveConf.getVar(ConfVars.HIVE_SERVER2_USE_SSL));
+ Boolean.toString(hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_USE_SSL)));
}
/**
@@ -443,6 +447,17 @@ public class HiveServer2 extends CompositeService {
@Override
public synchronized void start() {
super.start();
+ // If we're supporting dynamic service discovery, we'll add the service uri for this
+ // HiveServer2 instance to Zookeeper as a znode.
+ HiveConf hiveConf = this.getHiveConf();
+ if (hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_SUPPORT_DYNAMIC_SERVICE_DISCOVERY)) {
+ try {
+ addServerInstanceToZooKeeper(hiveConf);
+ } catch (Exception e) {
+ LOG.error("Error adding this HiveServer2 instance to ZooKeeper: ", e);
+ throw new ServiceException(e);
+ }
+ }
if (webServer != null) {
try {
webServer.start();
@@ -533,12 +548,6 @@ public class HiveServer2 extends CompositeService {
"warned upon.", t);
}
- // If we're supporting dynamic service discovery, we'll add the service uri for this
- // HiveServer2 instance to Zookeeper as a znode.
- if (hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_SUPPORT_DYNAMIC_SERVICE_DISCOVERY)) {
- server.addServerInstanceToZooKeeper(hiveConf);
- }
-
if (sessionPool != null) {
sessionPool.startPool();
}
[26/50] [abbrv] hive git commit: HIVE-13111: Fix timestamp /
interval_day_time wrong results with HIVE-9862 (Matt McCline,
reviewed by Jason Dere)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/test/results/clientpositive/tez/vectorized_casts.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vectorized_casts.q.out b/ql/src/test/results/clientpositive/tez/vectorized_casts.q.out
index b7b17de..cf6f4c7 100644
--- a/ql/src/test/results/clientpositive/tez/vectorized_casts.q.out
+++ b/ql/src/test/results/clientpositive/tez/vectorized_casts.q.out
@@ -353,18 +353,18 @@ true NULL true true true NULL true false true true 11 NULL -64615982 1803053750
true NULL true true true NULL true false true true 8 NULL 890988972 -1862301000 8 NULL 1 15 NULL NULL 8 8 8 8.0 NULL 8.90988972E8 -1.862301E9 8.0 NULL 1.0 15.892 NULL NULL 8.9098899E8 NULL 1969-12-31 16:00:00.008 NULL 1970-01-10 23:29:48.972 1969-12-10 02:41:39 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL 890988972 -1862301000 8.0 NULL TRUE 0 1969-12-31 16:00:15.892 XylAH4 XylAH4 XylAH4 8.0 1.781977944E9 0.9893582466233818 8.90988973E8
true NULL true true true NULL true false true true 8 NULL 930867246 1205399250 8 NULL 1 15 NULL NULL 8 8 8 8.0 NULL 9.30867246E8 1.20539925E9 8.0 NULL 1.0 15.892 NULL NULL 9.3086726E8 NULL 1969-12-31 16:00:00.008 NULL 1970-01-11 10:34:27.246 1970-01-14 14:49:59.25 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL 930867246 1205399250 8.0 NULL TRUE 0 1969-12-31 16:00:15.892 c1V8o1A c1V8o1A c1V8o1A 8.0 1.861734492E9 0.9893582466233818 9.30867247E8
true true NULL true true true NULL false true NULL -14 -7196 NULL -1552199500 -14 -7196 NULL 11 NULL NULL -14 -14 -14 -14.0 -7196.0 NULL -1.5521995E9 -14.0 -7196.0 NULL 11.065 NULL NULL NULL -7196.0 1969-12-31 15:59:59.986 1969-12-31 15:59:52.804 NULL 1969-12-13 16:50:00.5 1969-12-31 15:59:46 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:11.065 NULL NULL -14 -7196 NULL -1552199500 -14.0 -7196.0 NULL 0 1969-12-31 16:00:11.065 NULL NULL NULL -14.0 NULL -0.9906073556948704 NULL
-true true NULL true true true NULL false true NULL -21 -7196 NULL 1542429000 -21 -7196 NULL -4 NULL NULL -21 -21 -21 -21.0 -7196.0 NULL 1.542429E9 -21.0 -7196.0 NULL -4.1 NULL NULL NULL -7196.0 1969-12-31 15:59:59.979 1969-12-31 15:59:52.804 NULL 1970-01-18 12:27:09 1969-12-31 15:59:39 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:55.9 NULL NULL -21 -7196 NULL 1542429000 -21.0 -7196.0 NULL 0 1969-12-31 15:59:55.9 NULL NULL NULL -21.0 NULL -0.8366556385360561 NULL
-true true NULL true true true NULL false true NULL -24 -7196 NULL 829111000 -24 -7196 NULL -6 NULL NULL -24 -24 -24 -24.0 -7196.0 NULL 8.29111E8 -24.0 -7196.0 NULL -6.855 NULL NULL NULL -7196.0 1969-12-31 15:59:59.976 1969-12-31 15:59:52.804 NULL 1970-01-10 06:18:31 1969-12-31 15:59:36 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:53.145 NULL NULL -24 -7196 NULL 829111000 -24.0 -7196.0 NULL 0 1969-12-31 15:59:53.145 NULL NULL NULL -24.0 NULL 0.9055783620066238 NULL
+true true NULL true true true NULL false true NULL -21 -7196 NULL 1542429000 -21 -7196 NULL -5 NULL NULL -21 -21 -21 -21.0 -7196.0 NULL 1.542429E9 -21.0 -7196.0 NULL -4.1 NULL NULL NULL -7196.0 1969-12-31 15:59:59.979 1969-12-31 15:59:52.804 NULL 1970-01-18 12:27:09 1969-12-31 15:59:39 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:55.9 NULL NULL -21 -7196 NULL 1542429000 -21.0 -7196.0 NULL 0 1969-12-31 15:59:55.9 NULL NULL NULL -21.0 NULL -0.8366556385360561 NULL
+true true NULL true true true NULL false true NULL -24 -7196 NULL 829111000 -24 -7196 NULL -7 NULL NULL -24 -24 -24 -24.0 -7196.0 NULL 8.29111E8 -24.0 -7196.0 NULL -6.855 NULL NULL NULL -7196.0 1969-12-31 15:59:59.976 1969-12-31 15:59:52.804 NULL 1970-01-10 06:18:31 1969-12-31 15:59:36 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:53.145 NULL NULL -24 -7196 NULL 829111000 -24.0 -7196.0 NULL 0 1969-12-31 15:59:53.145 NULL NULL NULL -24.0 NULL 0.9055783620066238 NULL
true true NULL true true true NULL false true NULL -30 -200 NULL 1429852250 -30 -200 NULL 12 NULL NULL -30 -30 -30 -30.0 -200.0 NULL 1.42985225E9 -30.0 -200.0 NULL 12.935 NULL NULL NULL -200.0 1969-12-31 15:59:59.97 1969-12-31 15:59:59.8 NULL 1970-01-17 05:10:52.25 1969-12-31 15:59:30 1969-12-31 15:56:40 NULL 1969-12-31 16:00:00 1969-12-31 16:00:12.935 NULL NULL -30 -200 NULL 1429852250 -30.0 -200.0 NULL 0 1969-12-31 16:00:12.935 NULL NULL NULL -30.0 NULL 0.9880316240928618 NULL
-true true NULL true true true NULL false true NULL -36 -200 NULL -2006216750 -36 -200 NULL -14 NULL NULL -36 -36 -36 -36.0 -200.0 NULL -2.00621675E9 -36.0 -200.0 NULL -14.252 NULL NULL NULL -200.0 1969-12-31 15:59:59.964 1969-12-31 15:59:59.8 NULL 1969-12-08 10:43:03.25 1969-12-31 15:59:24 1969-12-31 15:56:40 NULL 1969-12-31 16:00:00 1969-12-31 15:59:45.748 NULL NULL -36 -200 NULL -2006216750 -36.0 -200.0 NULL 0 1969-12-31 15:59:45.748 NULL NULL NULL -36.0 NULL 0.9917788534431158 NULL
-true true NULL true true true NULL false true NULL -36 -200 NULL 1599879000 -36 -200 NULL -6 NULL NULL -36 -36 -36 -36.0 -200.0 NULL 1.599879E9 -36.0 -200.0 NULL -6.183 NULL NULL NULL -200.0 1969-12-31 15:59:59.964 1969-12-31 15:59:59.8 NULL 1970-01-19 04:24:39 1969-12-31 15:59:24 1969-12-31 15:56:40 NULL 1969-12-31 16:00:00 1969-12-31 15:59:53.817 NULL NULL -36 -200 NULL 1599879000 -36.0 -200.0 NULL 0 1969-12-31 15:59:53.817 NULL NULL NULL -36.0 NULL 0.9917788534431158 NULL
-true true NULL true true true NULL false true NULL -38 15601 NULL -1858689000 -38 15601 NULL -1 NULL NULL -38 -38 -38 -38.0 15601.0 NULL -1.858689E9 -38.0 15601.0 NULL -1.386 NULL NULL NULL 15601.0 1969-12-31 15:59:59.962 1969-12-31 16:00:15.601 NULL 1969-12-10 03:41:51 1969-12-31 15:59:22 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 15:59:58.614 NULL NULL -38 15601 NULL -1858689000 -38.0 15601.0 NULL 0 1969-12-31 15:59:58.614 NULL NULL NULL -38.0 NULL -0.2963685787093853 NULL
+true true NULL true true true NULL false true NULL -36 -200 NULL -2006216750 -36 -200 NULL -15 NULL NULL -36 -36 -36 -36.0 -200.0 NULL -2.00621675E9 -36.0 -200.0 NULL -14.252 NULL NULL NULL -200.0 1969-12-31 15:59:59.964 1969-12-31 15:59:59.8 NULL 1969-12-08 10:43:03.25 1969-12-31 15:59:24 1969-12-31 15:56:40 NULL 1969-12-31 16:00:00 1969-12-31 15:59:45.748 NULL NULL -36 -200 NULL -2006216750 -36.0 -200.0 NULL 0 1969-12-31 15:59:45.748 NULL NULL NULL -36.0 NULL 0.9917788534431158 NULL
+true true NULL true true true NULL false true NULL -36 -200 NULL 1599879000 -36 -200 NULL -7 NULL NULL -36 -36 -36 -36.0 -200.0 NULL 1.599879E9 -36.0 -200.0 NULL -6.183 NULL NULL NULL -200.0 1969-12-31 15:59:59.964 1969-12-31 15:59:59.8 NULL 1970-01-19 04:24:39 1969-12-31 15:59:24 1969-12-31 15:56:40 NULL 1969-12-31 16:00:00 1969-12-31 15:59:53.817 NULL NULL -36 -200 NULL 1599879000 -36.0 -200.0 NULL 0 1969-12-31 15:59:53.817 NULL NULL NULL -36.0 NULL 0.9917788534431158 NULL
+true true NULL true true true NULL false true NULL -38 15601 NULL -1858689000 -38 15601 NULL -2 NULL NULL -38 -38 -38 -38.0 15601.0 NULL -1.858689E9 -38.0 15601.0 NULL -1.3860000000000001 NULL NULL NULL 15601.0 1969-12-31 15:59:59.962 1969-12-31 16:00:15.601 NULL 1969-12-10 03:41:51 1969-12-31 15:59:22 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 15:59:58.614 NULL NULL -38 15601 NULL -1858689000 -38.0 15601.0 NULL 0 1969-12-31 15:59:58.614 NULL NULL NULL -38.0 NULL -0.2963685787093853 NULL
true true NULL true true true NULL false true NULL -5 15601 NULL 612416000 -5 15601 NULL 4 NULL NULL -5 -5 -5 -5.0 15601.0 NULL 6.12416E8 -5.0 15601.0 NULL 4.679 NULL NULL NULL 15601.0 1969-12-31 15:59:59.995 1969-12-31 16:00:15.601 NULL 1970-01-07 18:06:56 1969-12-31 15:59:55 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 16:00:04.679 NULL NULL -5 15601 NULL 612416000 -5.0 15601.0 NULL 0 1969-12-31 16:00:04.679 NULL NULL NULL -5.0 NULL 0.9589242746631385 NULL
-true true NULL true true true NULL false true NULL -50 -7196 NULL -1031187250 -50 -7196 NULL -5 NULL NULL -50 -50 -50 -50.0 -7196.0 NULL -1.03118725E9 -50.0 -7196.0 NULL -5.267 NULL NULL NULL -7196.0 1969-12-31 15:59:59.95 1969-12-31 15:59:52.804 NULL 1969-12-19 17:33:32.75 1969-12-31 15:59:10 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:54.733 NULL NULL -50 -7196 NULL -1031187250 -50.0 -7196.0 NULL 0 1969-12-31 15:59:54.733 NULL NULL NULL -50.0 NULL 0.26237485370392877 NULL
+true true NULL true true true NULL false true NULL -50 -7196 NULL -1031187250 -50 -7196 NULL -6 NULL NULL -50 -50 -50 -50.0 -7196.0 NULL -1.03118725E9 -50.0 -7196.0 NULL -5.267 NULL NULL NULL -7196.0 1969-12-31 15:59:59.95 1969-12-31 15:59:52.804 NULL 1969-12-19 17:33:32.75 1969-12-31 15:59:10 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:54.733 NULL NULL -50 -7196 NULL -1031187250 -50.0 -7196.0 NULL 0 1969-12-31 15:59:54.733 NULL NULL NULL -50.0 NULL 0.26237485370392877 NULL
true true NULL true true true NULL false true NULL -59 -7196 NULL -1604890000 -59 -7196 NULL 13 NULL NULL -59 -59 -59 -59.0 -7196.0 NULL -1.60489E9 -59.0 -7196.0 NULL 13.15 NULL NULL NULL -7196.0 1969-12-31 15:59:59.941 1969-12-31 15:59:52.804 NULL 1969-12-13 02:11:50 1969-12-31 15:59:01 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:13.15 NULL NULL -59 -7196 NULL -1604890000 -59.0 -7196.0 NULL 0 1969-12-31 16:00:13.15 NULL NULL NULL -59.0 NULL -0.6367380071391379 NULL
-true true NULL true true true NULL false true NULL -60 -7196 NULL 1516314750 -60 -7196 NULL -7 NULL NULL -60 -60 -60 -60.0 -7196.0 NULL 1.51631475E9 -60.0 -7196.0 NULL -7.592 NULL NULL NULL -7196.0 1969-12-31 15:59:59.94 1969-12-31 15:59:52.804 NULL 1970-01-18 05:11:54.75 1969-12-31 15:59:00 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:52.408 NULL NULL -60 -7196 NULL 1516314750 -60.0 -7196.0 NULL 0 1969-12-31 15:59:52.408 NULL NULL NULL -60.0 NULL 0.3048106211022167 NULL
+true true NULL true true true NULL false true NULL -60 -7196 NULL 1516314750 -60 -7196 NULL -8 NULL NULL -60 -60 -60 -60.0 -7196.0 NULL 1.51631475E9 -60.0 -7196.0 NULL -7.592 NULL NULL NULL -7196.0 1969-12-31 15:59:59.94 1969-12-31 15:59:52.804 NULL 1970-01-18 05:11:54.75 1969-12-31 15:59:00 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:52.408 NULL NULL -60 -7196 NULL 1516314750 -60.0 -7196.0 NULL 0 1969-12-31 15:59:52.408 NULL NULL NULL -60.0 NULL 0.3048106211022167 NULL
true true NULL true true true NULL false true NULL -8 -7196 NULL -1849991500 -8 -7196 NULL 3 NULL NULL -8 -8 -8 -8.0 -7196.0 NULL -1.8499915E9 -8.0 -7196.0 NULL 3.136 NULL NULL NULL -7196.0 1969-12-31 15:59:59.992 1969-12-31 15:59:52.804 NULL 1969-12-10 06:06:48.5 1969-12-31 15:59:52 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:03.136 NULL NULL -8 -7196 NULL -1849991500 -8.0 -7196.0 NULL 0 1969-12-31 16:00:03.136 NULL NULL NULL -8.0 NULL -0.9893582466233818 NULL
-true true NULL true true true NULL false true NULL 20 15601 NULL -362433250 20 15601 NULL -14 NULL NULL 20 20 20 20.0 15601.0 NULL -3.6243325E8 20.0 15601.0 NULL -14.871 NULL NULL NULL 15601.0 1969-12-31 16:00:00.02 1969-12-31 16:00:15.601 NULL 1969-12-27 11:19:26.75 1969-12-31 16:00:20 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 15:59:45.129 NULL NULL 20 15601 NULL -362433250 20.0 15601.0 NULL 0 1969-12-31 15:59:45.129 NULL NULL NULL 20.0 NULL 0.9129452507276277 NULL
-true true NULL true true true NULL false true NULL 48 15601 NULL -795361000 48 15601 NULL -9 NULL NULL 48 48 48 48.0 15601.0 NULL -7.95361E8 48.0 15601.0 NULL -9.765 NULL NULL NULL 15601.0 1969-12-31 16:00:00.048 1969-12-31 16:00:15.601 NULL 1969-12-22 11:03:59 1969-12-31 16:00:48 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 15:59:50.235 NULL NULL 48 15601 NULL -795361000 48.0 15601.0 NULL 0 1969-12-31 15:59:50.235 NULL NULL NULL 48.0 NULL -0.7682546613236668 NULL
+true true NULL true true true NULL false true NULL 20 15601 NULL -362433250 20 15601 NULL -15 NULL NULL 20 20 20 20.0 15601.0 NULL -3.6243325E8 20.0 15601.0 NULL -14.871 NULL NULL NULL 15601.0 1969-12-31 16:00:00.02 1969-12-31 16:00:15.601 NULL 1969-12-27 11:19:26.75 1969-12-31 16:00:20 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 15:59:45.129 NULL NULL 20 15601 NULL -362433250 20.0 15601.0 NULL 0 1969-12-31 15:59:45.129 NULL NULL NULL 20.0 NULL 0.9129452507276277 NULL
+true true NULL true true true NULL false true NULL 48 15601 NULL -795361000 48 15601 NULL -10 NULL NULL 48 48 48 48.0 15601.0 NULL -7.95361E8 48.0 15601.0 NULL -9.765 NULL NULL NULL 15601.0 1969-12-31 16:00:00.048 1969-12-31 16:00:15.601 NULL 1969-12-22 11:03:59 1969-12-31 16:00:48 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 15:59:50.235 NULL NULL 48 15601 NULL -795361000 48.0 15601.0 NULL 0 1969-12-31 15:59:50.235 NULL NULL NULL 48.0 NULL -0.7682546613236668 NULL
true true NULL true true true NULL false true NULL 5 -7196 NULL -1015607500 5 -7196 NULL 10 NULL NULL 5 5 5 5.0 -7196.0 NULL -1.0156075E9 5.0 -7196.0 NULL 10.973 NULL NULL NULL -7196.0 1969-12-31 16:00:00.005 1969-12-31 15:59:52.804 NULL 1969-12-19 21:53:12.5 1969-12-31 16:00:05 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:10.973 NULL NULL 5 -7196 NULL -1015607500 5.0 -7196.0 NULL 0 1969-12-31 16:00:10.973 NULL NULL NULL 5.0 NULL -0.9589242746631385 NULL
true true NULL true true true NULL false true NULL 59 -7196 NULL -1137754500 59 -7196 NULL 10 NULL NULL 59 59 59 59.0 -7196.0 NULL -1.1377545E9 59.0 -7196.0 NULL 10.956 NULL NULL NULL -7196.0 1969-12-31 16:00:00.059 1969-12-31 15:59:52.804 NULL 1969-12-18 11:57:25.5 1969-12-31 16:00:59 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:10.956 NULL NULL 59 -7196 NULL -1137754500 59.0 -7196.0 NULL 0 1969-12-31 16:00:10.956 NULL NULL NULL 59.0 NULL 0.6367380071391379 NULL
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/test/results/clientpositive/tez/vectorized_timestamp.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vectorized_timestamp.q.out b/ql/src/test/results/clientpositive/tez/vectorized_timestamp.q.out
new file mode 100644
index 0000000..5382865
--- /dev/null
+++ b/ql/src/test/results/clientpositive/tez/vectorized_timestamp.q.out
@@ -0,0 +1,157 @@
+PREHOOK: query: DROP TABLE IF EXISTS test
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS test
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE test(ts TIMESTAMP) STORED AS ORC
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@test
+POSTHOOK: query: CREATE TABLE test(ts TIMESTAMP) STORED AS ORC
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@test
+PREHOOK: query: INSERT INTO TABLE test VALUES ('0001-01-01 00:00:00.000000000'), ('9999-12-31 23:59:59.999999999')
+PREHOOK: type: QUERY
+PREHOOK: Input: default@values__tmp__table__1
+PREHOOK: Output: default@test
+POSTHOOK: query: INSERT INTO TABLE test VALUES ('0001-01-01 00:00:00.000000000'), ('9999-12-31 23:59:59.999999999')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@values__tmp__table__1
+POSTHOOK: Output: default@test
+POSTHOOK: Lineage: test.ts EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: EXPLAIN
+SELECT ts FROM test
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN
+SELECT ts FROM test
+POSTHOOK: type: QUERY
+Plan optimized by CBO.
+
+Stage-0
+ Fetch Operator
+ limit:-1
+ Stage-1
+ Map 1
+ File Output Operator [FS_2]
+ Select Operator [SEL_1] (rows=2 width=40)
+ Output:["_col0"]
+ TableScan [TS_0] (rows=2 width=40)
+ default@test,test,Tbl:COMPLETE,Col:NONE,Output:["ts"]
+
+PREHOOK: query: SELECT ts FROM test
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT ts FROM test
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test
+#### A masked pattern was here ####
+0001-01-01 00:00:00
+9999-12-31 23:59:59.999999999
+PREHOOK: query: EXPLAIN
+SELECT MIN(ts), MAX(ts), MAX(ts) - MIN(ts) FROM test
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN
+SELECT MIN(ts), MAX(ts), MAX(ts) - MIN(ts) FROM test
+POSTHOOK: type: QUERY
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Reducer 2 <- Map 1 (SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:-1
+ Stage-1
+ Reducer 2
+ File Output Operator [FS_6]
+ Select Operator [SEL_5] (rows=1 width=80)
+ Output:["_col0","_col1","_col2"]
+ Group By Operator [GBY_4] (rows=1 width=80)
+ Output:["_col0","_col1"],aggregations:["min(VALUE._col0)","max(VALUE._col1)"]
+ <-Map 1 [SIMPLE_EDGE]
+ SHUFFLE [RS_3]
+ Group By Operator [GBY_2] (rows=1 width=80)
+ Output:["_col0","_col1"],aggregations:["min(ts)","max(ts)"]
+ Select Operator [SEL_1] (rows=2 width=40)
+ Output:["ts"]
+ TableScan [TS_0] (rows=2 width=40)
+ default@test,test,Tbl:COMPLETE,Col:NONE,Output:["ts"]
+
+PREHOOK: query: SELECT MIN(ts), MAX(ts), MAX(ts) - MIN(ts) FROM test
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT MIN(ts), MAX(ts), MAX(ts) - MIN(ts) FROM test
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test
+#### A masked pattern was here ####
+0001-01-01 00:00:00 9999-12-31 23:59:59.999999999 3652060 23:59:59.999999999
+PREHOOK: query: EXPLAIN
+SELECT ts FROM test
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN
+SELECT ts FROM test
+POSTHOOK: type: QUERY
+Plan optimized by CBO.
+
+Stage-0
+ Fetch Operator
+ limit:-1
+ Stage-1
+ Map 1 vectorized
+ File Output Operator [FS_4]
+ Select Operator [OP_3] (rows=2 width=40)
+ Output:["_col0"]
+ TableScan [TS_0] (rows=2 width=40)
+ default@test,test,Tbl:COMPLETE,Col:NONE,Output:["ts"]
+
+PREHOOK: query: SELECT ts FROM test
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT ts FROM test
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test
+#### A masked pattern was here ####
+0001-01-01 00:00:00
+9999-12-31 23:59:59.999999999
+PREHOOK: query: EXPLAIN
+SELECT MIN(ts), MAX(ts), MAX(ts) - MIN(ts) FROM test
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN
+SELECT MIN(ts), MAX(ts), MAX(ts) - MIN(ts) FROM test
+POSTHOOK: type: QUERY
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Reducer 2 <- Map 1 (SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:-1
+ Stage-1
+ Reducer 2 vectorized
+ File Output Operator [FS_6]
+ Select Operator [SEL_5] (rows=1 width=80)
+ Output:["_col0","_col1","_col2"]
+ Group By Operator [OP_9] (rows=1 width=80)
+ Output:["_col0","_col1"],aggregations:["min(VALUE._col0)","max(VALUE._col1)"]
+ <-Map 1 [SIMPLE_EDGE] vectorized
+ SHUFFLE [RS_3]
+ Group By Operator [OP_8] (rows=1 width=80)
+ Output:["_col0","_col1"],aggregations:["min(ts)","max(ts)"]
+ Select Operator [OP_7] (rows=2 width=40)
+ Output:["ts"]
+ TableScan [TS_0] (rows=2 width=40)
+ default@test,test,Tbl:COMPLETE,Col:NONE,Output:["ts"]
+
+PREHOOK: query: SELECT MIN(ts), MAX(ts), MAX(ts) - MIN(ts) FROM test
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT MIN(ts), MAX(ts), MAX(ts) - MIN(ts) FROM test
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test
+#### A masked pattern was here ####
+0001-01-01 00:00:00 9999-12-31 23:59:59.999999999 3652060 23:59:59.999999999
[19/50] [abbrv] hive git commit: HIVE-12960: Migrate Column Stats
Extrapolation and UniformDistribution to HBaseStore (Pengcheng Xiong,
reviewed by Ashutosh Chauhan)
Posted by jd...@apache.org.
HIVE-12960: Migrate Column Stats Extrapolation and UniformDistribution to HBaseStore (Pengcheng Xiong, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/96862093
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/96862093
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/96862093
Branch: refs/heads/llap
Commit: 968620932301dc64cd435292726943a6c0a42551
Parents: 3038b05
Author: Pengcheng Xiong <px...@apache.org>
Authored: Sun Mar 27 11:46:17 2016 -0700
Committer: Pengcheng Xiong <px...@apache.org>
Committed: Sun Mar 27 12:11:39 2016 -0700
----------------------------------------------------------------------
.../hive/metastore/StatObjectConverter.java | 2 +-
.../hadoop/hive/metastore/hbase/HBaseUtils.java | 8 +-
.../hadoop/hive/metastore/hbase/StatsCache.java | 20 +-
.../stats/BinaryColumnStatsAggregator.java | 43 +-
.../stats/BooleanColumnStatsAggregator.java | 42 +-
.../hbase/stats/ColumnStatsAggregator.java | 12 +-
.../stats/ColumnStatsAggregatorFactory.java | 8 +-
.../stats/DecimalColumnStatsAggregator.java | 340 ++++++++-
.../stats/DoubleColumnStatsAggregator.java | 307 +++++++-
.../hbase/stats/IExtrapolatePartStatus.java | 30 +
.../hbase/stats/LongColumnStatsAggregator.java | 305 +++++++-
.../stats/StringColumnStatsAggregator.java | 85 ++-
...stHBaseAggregateStatsCacheWithBitVector.java | 6 +-
.../TestHBaseAggregateStatsExtrapolation.java | 717 +++++++++++++++++++
.../TestHBaseAggregateStatsNDVUniformDist.java | 581 +++++++++++++++
.../clientpositive/tez/explainuser_1.q.out | 92 +--
16 files changed, 2454 insertions(+), 144 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/96862093/metastore/src/java/org/apache/hadoop/hive/metastore/StatObjectConverter.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/StatObjectConverter.java b/metastore/src/java/org/apache/hadoop/hive/metastore/StatObjectConverter.java
index b3ceff1..e119dd8 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/StatObjectConverter.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/StatObjectConverter.java
@@ -650,7 +650,7 @@ public class StatObjectConverter {
}
}
- private static Decimal createThriftDecimal(String s) {
+ public static Decimal createThriftDecimal(String s) {
BigDecimal d = new BigDecimal(s);
return new Decimal(ByteBuffer.wrap(d.unscaledValue().toByteArray()), (short)d.scale());
}
http://git-wip-us.apache.org/repos/asf/hive/blob/96862093/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseUtils.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseUtils.java
index 9ec7cd5..e0b449b 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseUtils.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseUtils.java
@@ -19,6 +19,8 @@
package org.apache.hadoop.hive.metastore.hbase;
import java.io.IOException;
+import java.math.BigDecimal;
+import java.math.BigInteger;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
@@ -88,7 +90,7 @@ import com.google.protobuf.InvalidProtocolBufferException;
/**
* Utility functions
*/
-class HBaseUtils {
+public class HBaseUtils {
final static Charset ENCODING = StandardCharsets.UTF_8;
final static char KEY_SEPARATOR = '\u0001';
@@ -1421,4 +1423,8 @@ class HBaseUtils {
b[7] = (byte)(v >>> 0);
return b;
}
+
+ public static double getDoubleValue(Decimal decimal) {
+ return new BigDecimal(new BigInteger(decimal.getUnscaled()), decimal.getScale()).doubleValue();
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/96862093/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/StatsCache.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/StatsCache.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/StatsCache.java
index f1d2e50..18f8afc 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/StatsCache.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/StatsCache.java
@@ -85,12 +85,12 @@ class StatsCache {
@Override
public AggrStats load(StatsCacheKey key) throws Exception {
int numBitVectors = HiveStatsUtils.getNumBitVectorsForNDVEstimation(conf);
+ boolean useDensityFunctionForNDVEstimation = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_METASTORE_STATS_NDV_DENSITY_FUNCTION);
HBaseReadWrite hrw = HBaseReadWrite.getInstance();
AggrStats aggrStats = hrw.getAggregatedStats(key.hashed);
if (aggrStats == null) {
misses.incr();
ColumnStatsAggregator aggregator = null;
- ColumnStatisticsObj statsObj = null;
aggrStats = new AggrStats();
LOG.debug("Unable to find aggregated stats for " + key.colName + ", aggregating");
List<ColumnStatistics> css = hrw.getPartitionStatistics(key.dbName, key.tableName,
@@ -98,19 +98,13 @@ class StatsCache {
Collections.singletonList(key.colName));
if (css != null && css.size() > 0) {
aggrStats.setPartsFound(css.size());
- for (ColumnStatistics cs : css) {
- for (ColumnStatisticsObj cso : cs.getStatsObj()) {
- if (statsObj == null) {
- statsObj = ColumnStatsAggregatorFactory.newColumnStaticsObj(key.colName,
- cso.getColType(), cso.getStatsData().getSetField());
- }
- if (aggregator == null) {
- aggregator = ColumnStatsAggregatorFactory.getColumnStatsAggregator(
- cso.getStatsData().getSetField(), numBitVectors);
- }
- aggregator.aggregate(statsObj, cso);
- }
+ if (aggregator == null) {
+ aggregator = ColumnStatsAggregatorFactory.getColumnStatsAggregator(css.iterator()
+ .next().getStatsObj().iterator().next().getStatsData().getSetField(),
+ numBitVectors, useDensityFunctionForNDVEstimation);
}
+ ColumnStatisticsObj statsObj = aggregator
+ .aggregate(key.colName, key.partNames, css);
aggrStats.addToColStats(statsObj);
me.put(key, aggrStats);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/96862093/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/BinaryColumnStatsAggregator.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/BinaryColumnStatsAggregator.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/BinaryColumnStatsAggregator.java
index 40340dd..d81d612 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/BinaryColumnStatsAggregator.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/BinaryColumnStatsAggregator.java
@@ -19,17 +19,46 @@
package org.apache.hadoop.hive.metastore.hbase.stats;
+import java.util.List;
+
import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.MetaException;
-public class BinaryColumnStatsAggregator extends ColumnStatsAggregator{
+public class BinaryColumnStatsAggregator extends ColumnStatsAggregator {
@Override
- public void aggregate(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
- BinaryColumnStatsData aggregateData = aggregateColStats.getStatsData().getBinaryStats();
- BinaryColumnStatsData newData = newColStats.getStatsData().getBinaryStats();
- aggregateData.setMaxColLen(Math.max(aggregateData.getMaxColLen(), newData.getMaxColLen()));
- aggregateData.setAvgColLen(Math.max(aggregateData.getAvgColLen(), newData.getAvgColLen()));
- aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
+ public ColumnStatisticsObj aggregate(String colName, List<String> partNames,
+ List<ColumnStatistics> css) throws MetaException {
+ ColumnStatisticsObj statsObj = null;
+ BinaryColumnStatsData aggregateData = null;
+ String colType = null;
+ for (ColumnStatistics cs : css) {
+ if (cs.getStatsObjSize() != 1) {
+ throw new MetaException(
+ "The number of columns should be exactly one in aggrStats, but found "
+ + cs.getStatsObjSize());
+ }
+ ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
+ if (statsObj == null) {
+ colType = cso.getColType();
+ statsObj = ColumnStatsAggregatorFactory.newColumnStaticsObj(colName, colType, cso
+ .getStatsData().getSetField());
+ }
+ BinaryColumnStatsData newData = cso.getStatsData().getBinaryStats();
+ if (aggregateData == null) {
+ aggregateData = newData.deepCopy();
+ } else {
+ aggregateData.setMaxColLen(Math.max(aggregateData.getMaxColLen(), newData.getMaxColLen()));
+ aggregateData.setAvgColLen(Math.max(aggregateData.getAvgColLen(), newData.getAvgColLen()));
+ aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
+ }
+ }
+ ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData();
+ columnStatisticsData.setBinaryStats(aggregateData);
+ statsObj.setStatsData(columnStatisticsData);
+ return statsObj;
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/96862093/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/BooleanColumnStatsAggregator.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/BooleanColumnStatsAggregator.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/BooleanColumnStatsAggregator.java
index 735d965..e796df2 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/BooleanColumnStatsAggregator.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/BooleanColumnStatsAggregator.java
@@ -19,17 +19,47 @@
package org.apache.hadoop.hive.metastore.hbase.stats;
+import java.util.List;
+
import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.MetaException;
public class BooleanColumnStatsAggregator extends ColumnStatsAggregator {
@Override
- public void aggregate(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
- BooleanColumnStatsData aggregateData = aggregateColStats.getStatsData().getBooleanStats();
- BooleanColumnStatsData newData = newColStats.getStatsData().getBooleanStats();
- aggregateData.setNumTrues(aggregateData.getNumTrues() + newData.getNumTrues());
- aggregateData.setNumFalses(aggregateData.getNumFalses() + newData.getNumFalses());
- aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
+ public ColumnStatisticsObj aggregate(String colName, List<String> partNames,
+ List<ColumnStatistics> css) throws MetaException {
+ ColumnStatisticsObj statsObj = null;
+ BooleanColumnStatsData aggregateData = null;
+ String colType = null;
+ for (ColumnStatistics cs : css) {
+ if (cs.getStatsObjSize() != 1) {
+ throw new MetaException(
+ "The number of columns should be exactly one in aggrStats, but found "
+ + cs.getStatsObjSize());
+ }
+ ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
+ if (statsObj == null) {
+ colType = cso.getColType();
+ statsObj = ColumnStatsAggregatorFactory.newColumnStaticsObj(colName, colType, cso
+ .getStatsData().getSetField());
+ }
+ BooleanColumnStatsData newData = cso.getStatsData().getBooleanStats();
+ if (aggregateData == null) {
+ aggregateData = newData.deepCopy();
+ } else {
+ aggregateData.setNumTrues(aggregateData.getNumTrues() + newData.getNumTrues());
+ aggregateData.setNumFalses(aggregateData.getNumFalses() + newData.getNumFalses());
+ aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
+ }
+ }
+ ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData();
+ columnStatisticsData.setBooleanStats(aggregateData);
+ statsObj.setStatsData(columnStatisticsData);
+ return statsObj;
}
+
}
http://git-wip-us.apache.org/repos/asf/hive/blob/96862093/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/ColumnStatsAggregator.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/ColumnStatsAggregator.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/ColumnStatsAggregator.java
index 694e53b..31955b4 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/ColumnStatsAggregator.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/ColumnStatsAggregator.java
@@ -19,10 +19,16 @@
package org.apache.hadoop.hive.metastore.hbase.stats;
-import org.apache.hadoop.hive.metastore.NumDistinctValueEstimator;
+import java.util.List;
+
+import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.MetaException;
public abstract class ColumnStatsAggregator {
- NumDistinctValueEstimator ndvEstimator = null;
- public abstract void aggregate(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats);
+ public int numBitVectors;
+ public boolean useDensityFunctionForNDVEstimation;
+
+ public abstract ColumnStatisticsObj aggregate(String colName, List<String> partNames,
+ List<ColumnStatistics> css) throws MetaException;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/96862093/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/ColumnStatsAggregatorFactory.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/ColumnStatsAggregatorFactory.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/ColumnStatsAggregatorFactory.java
index 8eb127b..daf8569 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/ColumnStatsAggregatorFactory.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/ColumnStatsAggregatorFactory.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.hive.metastore.hbase.stats;
-import org.apache.hadoop.hive.metastore.NumDistinctValueEstimator;
import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData;
import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
@@ -35,7 +34,7 @@ public class ColumnStatsAggregatorFactory {
private ColumnStatsAggregatorFactory() {
}
- public static ColumnStatsAggregator getColumnStatsAggregator(_Fields type, int numBitVectors) {
+ public static ColumnStatsAggregator getColumnStatsAggregator(_Fields type, int numBitVectors, boolean useDensityFunctionForNDVEstimation) {
ColumnStatsAggregator agg;
switch (type) {
case BOOLEAN_STATS:
@@ -59,9 +58,8 @@ public class ColumnStatsAggregatorFactory {
default:
throw new RuntimeException("Woh, bad. Unknown stats type " + type.toString());
}
- if (numBitVectors > 0) {
- agg.ndvEstimator = new NumDistinctValueEstimator(numBitVectors);
- }
+ agg.numBitVectors = numBitVectors;
+ agg.useDensityFunctionForNDVEstimation = useDensityFunctionForNDVEstimation;
return agg;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/96862093/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/DecimalColumnStatsAggregator.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/DecimalColumnStatsAggregator.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/DecimalColumnStatsAggregator.java
index 50f4325..36b2c9c 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/DecimalColumnStatsAggregator.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/DecimalColumnStatsAggregator.java
@@ -19,33 +19,333 @@
package org.apache.hadoop.hive.metastore.hbase.stats;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
import org.apache.hadoop.hive.metastore.NumDistinctValueEstimator;
+import org.apache.hadoop.hive.metastore.StatObjectConverter;
+import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.Decimal;
import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.hbase.HBaseUtils;
-public class DecimalColumnStatsAggregator extends ColumnStatsAggregator {
+public class DecimalColumnStatsAggregator extends ColumnStatsAggregator implements
+ IExtrapolatePartStatus {
@Override
- public void aggregate(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
- DecimalColumnStatsData aggregateData = aggregateColStats.getStatsData().getDecimalStats();
- DecimalColumnStatsData newData = newColStats.getStatsData().getDecimalStats();
- Decimal lowValue = aggregateData.getLowValue() != null
- && (aggregateData.getLowValue().compareTo(newData.getLowValue()) > 0) ? aggregateData
- .getLowValue() : newData.getLowValue();
- aggregateData.setLowValue(lowValue);
- Decimal highValue = aggregateData.getHighValue() != null
- && (aggregateData.getHighValue().compareTo(newData.getHighValue()) > 0) ? aggregateData
- .getHighValue() : newData.getHighValue();
- aggregateData.setHighValue(highValue);
- aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
- if (ndvEstimator == null || !newData.isSetBitVectors() || newData.getBitVectors().length() == 0) {
- aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
+ public ColumnStatisticsObj aggregate(String colName, List<String> partNames,
+ List<ColumnStatistics> css) throws MetaException {
+ ColumnStatisticsObj statsObj = null;
+
+ // check if all the ColumnStatisticsObjs contain stats and all the ndv are
+ // bitvectors
+ boolean doAllPartitionContainStats = partNames.size() == css.size();
+ boolean isNDVBitVectorSet = true;
+ String colType = null;
+ for (ColumnStatistics cs : css) {
+ if (cs.getStatsObjSize() != 1) {
+ throw new MetaException(
+ "The number of columns should be exactly one in aggrStats, but found "
+ + cs.getStatsObjSize());
+ }
+ ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
+ if (statsObj == null) {
+ colType = cso.getColType();
+ statsObj = ColumnStatsAggregatorFactory.newColumnStaticsObj(colName, colType, cso
+ .getStatsData().getSetField());
+ }
+ if (numBitVectors <= 0 || !cso.getStatsData().getDecimalStats().isSetBitVectors()
+ || cso.getStatsData().getDecimalStats().getBitVectors().length() == 0) {
+ isNDVBitVectorSet = false;
+ break;
+ }
+ }
+ ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData();
+ if (doAllPartitionContainStats || css.size() < 2) {
+ DecimalColumnStatsData aggregateData = null;
+ long lowerBound = 0;
+ long higherBound = 0;
+ double densityAvgSum = 0.0;
+ NumDistinctValueEstimator ndvEstimator = null;
+ if (isNDVBitVectorSet) {
+ ndvEstimator = new NumDistinctValueEstimator(numBitVectors);
+ }
+ for (ColumnStatistics cs : css) {
+ ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
+ DecimalColumnStatsData newData = cso.getStatsData().getDecimalStats();
+ if (useDensityFunctionForNDVEstimation) {
+ lowerBound = Math.max(lowerBound, newData.getNumDVs());
+ higherBound += newData.getNumDVs();
+ densityAvgSum += (HBaseUtils.getDoubleValue(newData.getHighValue()) - HBaseUtils
+ .getDoubleValue(newData.getLowValue())) / newData.getNumDVs();
+ }
+ if (isNDVBitVectorSet) {
+ ndvEstimator.mergeEstimators(new NumDistinctValueEstimator(newData.getBitVectors(),
+ ndvEstimator.getnumBitVectors()));
+ }
+ if (aggregateData == null) {
+ aggregateData = newData.deepCopy();
+ } else {
+ if (HBaseUtils.getDoubleValue(aggregateData.getLowValue()) < HBaseUtils
+ .getDoubleValue(newData.getLowValue())) {
+ aggregateData.setLowValue(aggregateData.getLowValue());
+ } else {
+ aggregateData.setLowValue(newData.getLowValue());
+ }
+ if (HBaseUtils.getDoubleValue(aggregateData.getHighValue()) > HBaseUtils
+ .getDoubleValue(newData.getHighValue())) {
+ aggregateData.setHighValue(aggregateData.getHighValue());
+ } else {
+ aggregateData.setHighValue(newData.getHighValue());
+ }
+ aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
+ aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
+ }
+ }
+ if (isNDVBitVectorSet) {
+ // if all the ColumnStatisticsObjs contain bitvectors, we do not need to
+ // use uniform distribution assumption because we can merge bitvectors
+ // to get a good estimation.
+ aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
+ } else {
+ if (useDensityFunctionForNDVEstimation) {
+ // We have estimation, lowerbound and higherbound. We use estimation
+ // if it is between lowerbound and higherbound.
+ double densityAvg = densityAvgSum / partNames.size();
+ long estimation = (long) ((HBaseUtils.getDoubleValue(aggregateData.getHighValue()) - HBaseUtils
+ .getDoubleValue(aggregateData.getLowValue())) / densityAvg);
+ if (estimation < lowerBound) {
+ aggregateData.setNumDVs(lowerBound);
+ } else if (estimation > higherBound) {
+ aggregateData.setNumDVs(higherBound);
+ } else {
+ aggregateData.setNumDVs(estimation);
+ }
+ } else {
+ // Without useDensityFunctionForNDVEstimation, we just use the
+ // default one, which is the max of all the partitions and it is
+ // already done.
+ }
+ }
+ columnStatisticsData.setDecimalStats(aggregateData);
+ } else {
+ // we need extrapolation
+ Map<String, Integer> indexMap = new HashMap<String, Integer>();
+ for (int index = 0; index < partNames.size(); index++) {
+ indexMap.put(partNames.get(index), index);
+ }
+ Map<String, Double> adjustedIndexMap = new HashMap<String, Double>();
+ Map<String, ColumnStatisticsData> adjustedStatsMap = new HashMap<String, ColumnStatisticsData>();
+ // while we scan the css, we also get the densityAvg, lowerbound and
+ // higerbound when useDensityFunctionForNDVEstimation is true.
+ double densityAvgSum = 0.0;
+ if (!isNDVBitVectorSet) {
+ // if not every partition uses bitvector for ndv, we just fall back to
+ // the traditional extrapolation methods.
+ for (ColumnStatistics cs : css) {
+ String partName = cs.getStatsDesc().getPartName();
+ ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
+ DecimalColumnStatsData newData = cso.getStatsData().getDecimalStats();
+ if (useDensityFunctionForNDVEstimation) {
+ densityAvgSum += (HBaseUtils.getDoubleValue(newData.getHighValue()) - HBaseUtils
+ .getDoubleValue(newData.getLowValue())) / newData.getNumDVs();
+ }
+ adjustedIndexMap.put(partName, (double) indexMap.get(partName));
+ adjustedStatsMap.put(partName, cso.getStatsData());
+ }
+ } else {
+ // we first merge all the adjacent bitvectors that we could merge and
+ // derive new partition names and index.
+ NumDistinctValueEstimator ndvEstimator = new NumDistinctValueEstimator(numBitVectors);
+ StringBuilder pseudoPartName = new StringBuilder();
+ double pseudoIndexSum = 0;
+ int length = 0;
+ int curIndex = -1;
+ DecimalColumnStatsData aggregateData = null;
+ for (ColumnStatistics cs : css) {
+ String partName = cs.getStatsDesc().getPartName();
+ ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
+ DecimalColumnStatsData newData = cso.getStatsData().getDecimalStats();
+ // newData.isSetBitVectors() should be true for sure because we
+ // already checked it before.
+ if (indexMap.get(partName) != curIndex) {
+ // There is bitvector, but it is not adjacent to the previous ones.
+ if (length > 0) {
+ // we have to set ndv
+ adjustedIndexMap.put(pseudoPartName.toString(), pseudoIndexSum / length);
+ aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
+ ColumnStatisticsData csd = new ColumnStatisticsData();
+ csd.setDecimalStats(aggregateData);
+ adjustedStatsMap.put(pseudoPartName.toString(), csd);
+ if (useDensityFunctionForNDVEstimation) {
+ densityAvgSum += (HBaseUtils.getDoubleValue(aggregateData.getHighValue()) - HBaseUtils
+ .getDoubleValue(aggregateData.getLowValue())) / aggregateData.getNumDVs();
+ }
+ // reset everything
+ pseudoPartName = new StringBuilder();
+ pseudoIndexSum = 0;
+ length = 0;
+ }
+ aggregateData = null;
+ }
+ curIndex = indexMap.get(partName);
+ pseudoPartName.append(partName);
+ pseudoIndexSum += curIndex;
+ length++;
+ curIndex++;
+ if (aggregateData == null) {
+ aggregateData = newData.deepCopy();
+ } else {
+ if (HBaseUtils.getDoubleValue(aggregateData.getLowValue()) < HBaseUtils
+ .getDoubleValue(newData.getLowValue())) {
+ aggregateData.setLowValue(aggregateData.getLowValue());
+ } else {
+ aggregateData.setLowValue(newData.getLowValue());
+ }
+ if (HBaseUtils.getDoubleValue(aggregateData.getHighValue()) > HBaseUtils
+ .getDoubleValue(newData.getHighValue())) {
+ aggregateData.setHighValue(aggregateData.getHighValue());
+ } else {
+ aggregateData.setHighValue(newData.getHighValue());
+ }
+ aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
+ }
+ ndvEstimator.mergeEstimators(new NumDistinctValueEstimator(newData.getBitVectors(),
+ ndvEstimator.getnumBitVectors()));
+ }
+ if (length > 0) {
+ // we have to set ndv
+ adjustedIndexMap.put(pseudoPartName.toString(), pseudoIndexSum / length);
+ aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
+ ColumnStatisticsData csd = new ColumnStatisticsData();
+ csd.setDecimalStats(aggregateData);
+ adjustedStatsMap.put(pseudoPartName.toString(), csd);
+ if (useDensityFunctionForNDVEstimation) {
+ densityAvgSum += (HBaseUtils.getDoubleValue(aggregateData.getHighValue()) - HBaseUtils
+ .getDoubleValue(aggregateData.getLowValue())) / aggregateData.getNumDVs();
+ }
+ }
+ }
+ extrapolate(columnStatisticsData, partNames.size(), css.size(), adjustedIndexMap,
+ adjustedStatsMap, densityAvgSum / adjustedStatsMap.size());
+ }
+ statsObj.setStatsData(columnStatisticsData);
+ return statsObj;
+ }
+
+ @Override
+ public void extrapolate(ColumnStatisticsData extrapolateData, int numParts,
+ int numPartsWithStats, Map<String, Double> adjustedIndexMap,
+ Map<String, ColumnStatisticsData> adjustedStatsMap, double densityAvg) {
+ int rightBorderInd = numParts;
+ DecimalColumnStatsData extrapolateDecimalData = new DecimalColumnStatsData();
+ Map<String, DecimalColumnStatsData> extractedAdjustedStatsMap = new HashMap<>();
+ for (Map.Entry<String, ColumnStatisticsData> entry : adjustedStatsMap.entrySet()) {
+ extractedAdjustedStatsMap.put(entry.getKey(), entry.getValue().getDecimalStats());
+ }
+ List<Map.Entry<String, DecimalColumnStatsData>> list = new LinkedList<Map.Entry<String, DecimalColumnStatsData>>(
+ extractedAdjustedStatsMap.entrySet());
+ // get the lowValue
+ Collections.sort(list, new Comparator<Map.Entry<String, DecimalColumnStatsData>>() {
+ public int compare(Map.Entry<String, DecimalColumnStatsData> o1,
+ Map.Entry<String, DecimalColumnStatsData> o2) {
+ return o1.getValue().getLowValue().compareTo(o2.getValue().getLowValue());
+ }
+ });
+ double minInd = adjustedIndexMap.get(list.get(0).getKey());
+ double maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
+ double lowValue = 0;
+ double min = HBaseUtils.getDoubleValue(list.get(0).getValue().getLowValue());
+ double max = HBaseUtils.getDoubleValue(list.get(list.size() - 1).getValue().getLowValue());
+ if (minInd == maxInd) {
+ lowValue = min;
+ } else if (minInd < maxInd) {
+ // left border is the min
+ lowValue = (max - (max - min) * maxInd / (maxInd - minInd));
+ } else {
+ // right border is the min
+ lowValue = (max - (max - min) * (rightBorderInd - maxInd) / (minInd - maxInd));
+ }
+
+ // get the highValue
+ Collections.sort(list, new Comparator<Map.Entry<String, DecimalColumnStatsData>>() {
+ public int compare(Map.Entry<String, DecimalColumnStatsData> o1,
+ Map.Entry<String, DecimalColumnStatsData> o2) {
+ return o1.getValue().getHighValue().compareTo(o2.getValue().getHighValue());
+ }
+ });
+ minInd = adjustedIndexMap.get(list.get(0).getKey());
+ maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
+ double highValue = 0;
+ min = HBaseUtils.getDoubleValue(list.get(0).getValue().getHighValue());
+ max = HBaseUtils.getDoubleValue(list.get(list.size() - 1).getValue().getHighValue());
+ if (minInd == maxInd) {
+ highValue = min;
+ } else if (minInd < maxInd) {
+ // right border is the max
+ highValue = (min + (max - min) * (rightBorderInd - minInd) / (maxInd - minInd));
+ } else {
+ // left border is the max
+ highValue = (min + (max - min) * minInd / (minInd - maxInd));
+ }
+
+ // get the #nulls
+ long numNulls = 0;
+ for (Map.Entry<String, DecimalColumnStatsData> entry : extractedAdjustedStatsMap.entrySet()) {
+ numNulls += entry.getValue().getNumNulls();
+ }
+ // we scale up sumNulls based on the number of partitions
+ numNulls = numNulls * numParts / numPartsWithStats;
+
+ // get the ndv
+ long ndv = 0;
+ long ndvMin = 0;
+ long ndvMax = 0;
+ Collections.sort(list, new Comparator<Map.Entry<String, DecimalColumnStatsData>>() {
+ public int compare(Map.Entry<String, DecimalColumnStatsData> o1,
+ Map.Entry<String, DecimalColumnStatsData> o2) {
+ return o1.getValue().getNumDVs() < o2.getValue().getNumDVs() ? -1 : 1;
+ }
+ });
+ long lowerBound = list.get(list.size() - 1).getValue().getNumDVs();
+ long higherBound = 0;
+ for (Map.Entry<String, DecimalColumnStatsData> entry : list) {
+ higherBound += entry.getValue().getNumDVs();
+ }
+ if (useDensityFunctionForNDVEstimation && densityAvg != 0.0) {
+ ndv = (long) ((highValue - lowValue) / densityAvg);
+ if (ndv < lowerBound) {
+ ndv = lowerBound;
+ } else if (ndv > higherBound) {
+ ndv = higherBound;
+ }
} else {
- ndvEstimator.mergeEstimators(new NumDistinctValueEstimator(newData.getBitVectors(),
- ndvEstimator.getnumBitVectors()));
- aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
- aggregateData.setBitVectors(ndvEstimator.serialize().toString());
+ minInd = adjustedIndexMap.get(list.get(0).getKey());
+ maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
+ ndvMin = list.get(0).getValue().getNumDVs();
+ ndvMax = list.get(list.size() - 1).getValue().getNumDVs();
+ if (minInd == maxInd) {
+ ndv = ndvMin;
+ } else if (minInd < maxInd) {
+ // right border is the max
+ ndv = (long) (ndvMin + (ndvMax - ndvMin) * (rightBorderInd - minInd) / (maxInd - minInd));
+ } else {
+ // left border is the max
+ ndv = (long) (ndvMin + (ndvMax - ndvMin) * minInd / (minInd - maxInd));
+ }
}
+ extrapolateDecimalData.setLowValue(StatObjectConverter.createThriftDecimal(String
+ .valueOf(lowValue)));
+ extrapolateDecimalData.setHighValue(StatObjectConverter.createThriftDecimal(String
+ .valueOf(highValue)));
+ extrapolateDecimalData.setNumNulls(numNulls);
+ extrapolateDecimalData.setNumDVs(ndv);
+ extrapolateData.setDecimalStats(extrapolateDecimalData);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/96862093/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/DoubleColumnStatsAggregator.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/DoubleColumnStatsAggregator.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/DoubleColumnStatsAggregator.java
index d945ec2..a88ef84 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/DoubleColumnStatsAggregator.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/DoubleColumnStatsAggregator.java
@@ -19,26 +19,307 @@
package org.apache.hadoop.hive.metastore.hbase.stats;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
import org.apache.hadoop.hive.metastore.NumDistinctValueEstimator;
+import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.MetaException;
-public class DoubleColumnStatsAggregator extends ColumnStatsAggregator {
+public class DoubleColumnStatsAggregator extends ColumnStatsAggregator implements
+ IExtrapolatePartStatus {
@Override
- public void aggregate(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
- DoubleColumnStatsData aggregateData = aggregateColStats.getStatsData().getDoubleStats();
- DoubleColumnStatsData newData = newColStats.getStatsData().getDoubleStats();
- aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue()));
- aggregateData.setHighValue(Math.max(aggregateData.getHighValue(), newData.getHighValue()));
- aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
- if (ndvEstimator == null || !newData.isSetBitVectors() || newData.getBitVectors().length() == 0) {
- aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
+ public ColumnStatisticsObj aggregate(String colName, List<String> partNames,
+ List<ColumnStatistics> css) throws MetaException {
+ ColumnStatisticsObj statsObj = null;
+
+ // check if all the ColumnStatisticsObjs contain stats and all the ndv are
+ // bitvectors
+ boolean doAllPartitionContainStats = partNames.size() == css.size();
+ boolean isNDVBitVectorSet = true;
+ String colType = null;
+ for (ColumnStatistics cs : css) {
+ if (cs.getStatsObjSize() != 1) {
+ throw new MetaException(
+ "The number of columns should be exactly one in aggrStats, but found "
+ + cs.getStatsObjSize());
+ }
+ ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
+ if (statsObj == null) {
+ colType = cso.getColType();
+ statsObj = ColumnStatsAggregatorFactory.newColumnStaticsObj(colName, colType, cso
+ .getStatsData().getSetField());
+ }
+ if (numBitVectors <= 0 || !cso.getStatsData().getDoubleStats().isSetBitVectors()
+ || cso.getStatsData().getDoubleStats().getBitVectors().length() == 0) {
+ isNDVBitVectorSet = false;
+ break;
+ }
+ }
+ ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData();
+ if (doAllPartitionContainStats || css.size() < 2) {
+ DoubleColumnStatsData aggregateData = null;
+ long lowerBound = 0;
+ long higherBound = 0;
+ double densityAvgSum = 0.0;
+ NumDistinctValueEstimator ndvEstimator = null;
+ if (isNDVBitVectorSet) {
+ ndvEstimator = new NumDistinctValueEstimator(numBitVectors);
+ }
+ for (ColumnStatistics cs : css) {
+ ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
+ DoubleColumnStatsData newData = cso.getStatsData().getDoubleStats();
+ if (useDensityFunctionForNDVEstimation) {
+ lowerBound = Math.max(lowerBound, newData.getNumDVs());
+ higherBound += newData.getNumDVs();
+ densityAvgSum += (newData.getHighValue() - newData.getLowValue()) / newData.getNumDVs();
+ }
+ if (isNDVBitVectorSet) {
+ ndvEstimator.mergeEstimators(new NumDistinctValueEstimator(newData.getBitVectors(),
+ ndvEstimator.getnumBitVectors()));
+ }
+ if (aggregateData == null) {
+ aggregateData = newData.deepCopy();
+ } else {
+ aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue()));
+ aggregateData
+ .setHighValue(Math.max(aggregateData.getHighValue(), newData.getHighValue()));
+ aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
+ aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
+ }
+ }
+ if (isNDVBitVectorSet) {
+ // if all the ColumnStatisticsObjs contain bitvectors, we do not need to
+ // use uniform distribution assumption because we can merge bitvectors
+ // to get a good estimation.
+ aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
+ } else {
+ if (useDensityFunctionForNDVEstimation) {
+ // We have estimation, lowerbound and higherbound. We use estimation
+ // if it is between lowerbound and higherbound.
+ double densityAvg = densityAvgSum / partNames.size();
+ long estimation = (long) ((aggregateData.getHighValue() - aggregateData.getLowValue()) / densityAvg);
+ if (estimation < lowerBound) {
+ aggregateData.setNumDVs(lowerBound);
+ } else if (estimation > higherBound) {
+ aggregateData.setNumDVs(higherBound);
+ } else {
+ aggregateData.setNumDVs(estimation);
+ }
+ } else {
+ // Without useDensityFunctionForNDVEstimation, we just use the
+ // default one, which is the max of all the partitions and it is
+ // already done.
+ }
+ }
+ columnStatisticsData.setDoubleStats(aggregateData);
+ } else {
+ // we need extrapolation
+ Map<String, Integer> indexMap = new HashMap<String, Integer>();
+ for (int index = 0; index < partNames.size(); index++) {
+ indexMap.put(partNames.get(index), index);
+ }
+ Map<String, Double> adjustedIndexMap = new HashMap<String, Double>();
+ Map<String, ColumnStatisticsData> adjustedStatsMap = new HashMap<String, ColumnStatisticsData>();
+ // while we scan the css, we also get the densityAvg, lowerbound and
+ // higerbound when useDensityFunctionForNDVEstimation is true.
+ double densityAvgSum = 0.0;
+ if (!isNDVBitVectorSet) {
+ // if not every partition uses bitvector for ndv, we just fall back to
+ // the traditional extrapolation methods.
+ for (ColumnStatistics cs : css) {
+ String partName = cs.getStatsDesc().getPartName();
+ ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
+ DoubleColumnStatsData newData = cso.getStatsData().getDoubleStats();
+ if (useDensityFunctionForNDVEstimation) {
+ densityAvgSum += (newData.getHighValue() - newData.getLowValue()) / newData.getNumDVs();
+ }
+ adjustedIndexMap.put(partName, (double) indexMap.get(partName));
+ adjustedStatsMap.put(partName, cso.getStatsData());
+ }
+ } else {
+ // we first merge all the adjacent bitvectors that we could merge and
+ // derive new partition names and index.
+ NumDistinctValueEstimator ndvEstimator = new NumDistinctValueEstimator(numBitVectors);
+ StringBuilder pseudoPartName = new StringBuilder();
+ double pseudoIndexSum = 0;
+ int length = 0;
+ int curIndex = -1;
+ DoubleColumnStatsData aggregateData = null;
+ for (ColumnStatistics cs : css) {
+ String partName = cs.getStatsDesc().getPartName();
+ ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
+ DoubleColumnStatsData newData = cso.getStatsData().getDoubleStats();
+ // newData.isSetBitVectors() should be true for sure because we
+ // already checked it before.
+ if (indexMap.get(partName) != curIndex) {
+ // There is bitvector, but it is not adjacent to the previous ones.
+ if (length > 0) {
+ // we have to set ndv
+ adjustedIndexMap.put(pseudoPartName.toString(), pseudoIndexSum / length);
+ aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
+ ColumnStatisticsData csd = new ColumnStatisticsData();
+ csd.setDoubleStats(aggregateData);
+ adjustedStatsMap.put(pseudoPartName.toString(), csd);
+ if (useDensityFunctionForNDVEstimation) {
+ densityAvgSum += (aggregateData.getHighValue() - aggregateData.getLowValue()) / aggregateData.getNumDVs();
+ }
+ // reset everything
+ pseudoPartName = new StringBuilder();
+ pseudoIndexSum = 0;
+ length = 0;
+ }
+ aggregateData = null;
+ }
+ curIndex = indexMap.get(partName);
+ pseudoPartName.append(partName);
+ pseudoIndexSum += curIndex;
+ length++;
+ curIndex++;
+ if (aggregateData == null) {
+ aggregateData = newData.deepCopy();
+ } else {
+ aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue()));
+ aggregateData.setHighValue(Math.max(aggregateData.getHighValue(),
+ newData.getHighValue()));
+ aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
+ }
+ ndvEstimator.mergeEstimators(new NumDistinctValueEstimator(newData.getBitVectors(),
+ ndvEstimator.getnumBitVectors()));
+ }
+ if (length > 0) {
+ // we have to set ndv
+ adjustedIndexMap.put(pseudoPartName.toString(), pseudoIndexSum / length);
+ aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
+ ColumnStatisticsData csd = new ColumnStatisticsData();
+ csd.setDoubleStats(aggregateData);
+ adjustedStatsMap.put(pseudoPartName.toString(), csd);
+ if (useDensityFunctionForNDVEstimation) {
+ densityAvgSum += (aggregateData.getHighValue() - aggregateData.getLowValue()) / aggregateData.getNumDVs();
+ }
+ }
+ }
+ extrapolate(columnStatisticsData, partNames.size(), css.size(), adjustedIndexMap,
+ adjustedStatsMap, densityAvgSum / adjustedStatsMap.size());
+ }
+ statsObj.setStatsData(columnStatisticsData);
+ return statsObj;
+ }
+
+ @Override
+ public void extrapolate(ColumnStatisticsData extrapolateData, int numParts,
+ int numPartsWithStats, Map<String, Double> adjustedIndexMap,
+ Map<String, ColumnStatisticsData> adjustedStatsMap, double densityAvg) {
+ int rightBorderInd = numParts;
+ DoubleColumnStatsData extrapolateDoubleData = new DoubleColumnStatsData();
+ Map<String, DoubleColumnStatsData> extractedAdjustedStatsMap = new HashMap<>();
+ for (Map.Entry<String, ColumnStatisticsData> entry : adjustedStatsMap.entrySet()) {
+ extractedAdjustedStatsMap.put(entry.getKey(), entry.getValue().getDoubleStats());
+ }
+ List<Map.Entry<String, DoubleColumnStatsData>> list = new LinkedList<Map.Entry<String, DoubleColumnStatsData>>(
+ extractedAdjustedStatsMap.entrySet());
+ // get the lowValue
+ Collections.sort(list, new Comparator<Map.Entry<String, DoubleColumnStatsData>>() {
+ public int compare(Map.Entry<String, DoubleColumnStatsData> o1,
+ Map.Entry<String, DoubleColumnStatsData> o2) {
+ return o1.getValue().getLowValue() < o2.getValue().getLowValue() ? -1 : 1;
+ }
+ });
+ double minInd = adjustedIndexMap.get(list.get(0).getKey());
+ double maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
+ double lowValue = 0;
+ double min = list.get(0).getValue().getLowValue();
+ double max = list.get(list.size() - 1).getValue().getLowValue();
+ if (minInd == maxInd) {
+ lowValue = min;
+ } else if (minInd < maxInd) {
+ // left border is the min
+ lowValue = (max - (max - min) * maxInd / (maxInd - minInd));
+ } else {
+ // right border is the min
+ lowValue = (max - (max - min) * (rightBorderInd - maxInd) / (minInd - maxInd));
+ }
+
+ // get the highValue
+ Collections.sort(list, new Comparator<Map.Entry<String, DoubleColumnStatsData>>() {
+ public int compare(Map.Entry<String, DoubleColumnStatsData> o1,
+ Map.Entry<String, DoubleColumnStatsData> o2) {
+ return o1.getValue().getHighValue() < o2.getValue().getHighValue() ? -1 : 1;
+ }
+ });
+ minInd = adjustedIndexMap.get(list.get(0).getKey());
+ maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
+ double highValue = 0;
+ min = list.get(0).getValue().getHighValue();
+ max = list.get(list.size() - 1).getValue().getHighValue();
+ if (minInd == maxInd) {
+ highValue = min;
+ } else if (minInd < maxInd) {
+ // right border is the max
+ highValue = (min + (max - min) * (rightBorderInd - minInd) / (maxInd - minInd));
+ } else {
+ // left border is the max
+ highValue = (min + (max - min) * minInd / (minInd - maxInd));
+ }
+
+ // get the #nulls
+ long numNulls = 0;
+ for (Map.Entry<String, DoubleColumnStatsData> entry : extractedAdjustedStatsMap.entrySet()) {
+ numNulls += entry.getValue().getNumNulls();
+ }
+ // we scale up sumNulls based on the number of partitions
+ numNulls = numNulls * numParts / numPartsWithStats;
+
+ // get the ndv
+ long ndv = 0;
+ long ndvMin = 0;
+ long ndvMax = 0;
+ Collections.sort(list, new Comparator<Map.Entry<String, DoubleColumnStatsData>>() {
+ public int compare(Map.Entry<String, DoubleColumnStatsData> o1,
+ Map.Entry<String, DoubleColumnStatsData> o2) {
+ return o1.getValue().getNumDVs() < o2.getValue().getNumDVs() ? -1 : 1;
+ }
+ });
+ long lowerBound = list.get(list.size() - 1).getValue().getNumDVs();
+ long higherBound = 0;
+ for (Map.Entry<String, DoubleColumnStatsData> entry : list) {
+ higherBound += entry.getValue().getNumDVs();
+ }
+ if (useDensityFunctionForNDVEstimation && densityAvg != 0.0) {
+ ndv = (long) ((highValue - lowValue) / densityAvg);
+ if (ndv < lowerBound) {
+ ndv = lowerBound;
+ } else if (ndv > higherBound) {
+ ndv = higherBound;
+ }
} else {
- ndvEstimator.mergeEstimators(new NumDistinctValueEstimator(newData.getBitVectors(),
- ndvEstimator.getnumBitVectors()));
- aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
- aggregateData.setBitVectors(ndvEstimator.serialize().toString());
+ minInd = adjustedIndexMap.get(list.get(0).getKey());
+ maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
+ ndvMin = list.get(0).getValue().getNumDVs();
+ ndvMax = list.get(list.size() - 1).getValue().getNumDVs();
+ if (minInd == maxInd) {
+ ndv = ndvMin;
+ } else if (minInd < maxInd) {
+ // right border is the max
+ ndv = (long) (ndvMin + (ndvMax - ndvMin) * (rightBorderInd - minInd) / (maxInd - minInd));
+ } else {
+ // left border is the max
+ ndv = (long) (ndvMin + (ndvMax - ndvMin) * minInd / (minInd - maxInd));
+ }
}
+ extrapolateDoubleData.setLowValue(lowValue);
+ extrapolateDoubleData.setHighValue(highValue);
+ extrapolateDoubleData.setNumNulls(numNulls);
+ extrapolateDoubleData.setNumDVs(ndv);
+ extrapolateData.setDoubleStats(extrapolateDoubleData);
}
+
}
http://git-wip-us.apache.org/repos/asf/hive/blob/96862093/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/IExtrapolatePartStatus.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/IExtrapolatePartStatus.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/IExtrapolatePartStatus.java
new file mode 100644
index 0000000..99af060
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/IExtrapolatePartStatus.java
@@ -0,0 +1,30 @@
+package org.apache.hadoop.hive.metastore.hbase.stats;
+
+import java.util.Map;
+
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
+
+public interface IExtrapolatePartStatus {
+ // The following function will extrapolate the stats when the column stats of
+ // some partitions are missing.
+ /**
+ * @param extrapolateData
+ * it will carry back the specific stats, e.g., DOUBLE_STATS or
+ * LONG_STATS
+ * @param numParts
+ * the total number of partitions
+ * @param numPartsWithStats
+ * the number of partitions that have stats
+ * @param adjustedIndexMap
+ * the partition name to index map
+ * @param adjustedStatsMap
+ * the partition name to its stats map
+ * @param densityAvg
+ * the average of ndv density, which is useful when
+ * useDensityFunctionForNDVEstimation is true.
+ */
+ public abstract void extrapolate(ColumnStatisticsData extrapolateData, int numParts,
+ int numPartsWithStats, Map<String, Double> adjustedIndexMap,
+ Map<String, ColumnStatisticsData> adjustedStatsMap, double densityAvg);
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/96862093/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/LongColumnStatsAggregator.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/LongColumnStatsAggregator.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/LongColumnStatsAggregator.java
index 068dd00..8ac6561 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/LongColumnStatsAggregator.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/LongColumnStatsAggregator.java
@@ -19,26 +19,305 @@
package org.apache.hadoop.hive.metastore.hbase.stats;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
import org.apache.hadoop.hive.metastore.NumDistinctValueEstimator;
+import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.MetaException;
-public class LongColumnStatsAggregator extends ColumnStatsAggregator {
+public class LongColumnStatsAggregator extends ColumnStatsAggregator implements
+ IExtrapolatePartStatus {
@Override
- public void aggregate(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
- LongColumnStatsData aggregateData = aggregateColStats.getStatsData().getLongStats();
- LongColumnStatsData newData = newColStats.getStatsData().getLongStats();
- aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue()));
- aggregateData.setHighValue(Math.max(aggregateData.getHighValue(), newData.getHighValue()));
- aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
- if (ndvEstimator == null || !newData.isSetBitVectors() || newData.getBitVectors().length() == 0) {
- aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
+ public ColumnStatisticsObj aggregate(String colName, List<String> partNames,
+ List<ColumnStatistics> css) throws MetaException {
+ ColumnStatisticsObj statsObj = null;
+
+ // check if all the ColumnStatisticsObjs contain stats and all the ndv are
+ // bitvectors
+ boolean doAllPartitionContainStats = partNames.size() == css.size();
+ boolean isNDVBitVectorSet = true;
+ String colType = null;
+ for (ColumnStatistics cs : css) {
+ if (cs.getStatsObjSize() != 1) {
+ throw new MetaException(
+ "The number of columns should be exactly one in aggrStats, but found "
+ + cs.getStatsObjSize());
+ }
+ ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
+ if (statsObj == null) {
+ colType = cso.getColType();
+ statsObj = ColumnStatsAggregatorFactory.newColumnStaticsObj(colName, colType, cso
+ .getStatsData().getSetField());
+ }
+ if (numBitVectors <= 0 || !cso.getStatsData().getLongStats().isSetBitVectors()
+ || cso.getStatsData().getLongStats().getBitVectors().length() == 0) {
+ isNDVBitVectorSet = false;
+ break;
+ }
+ }
+ ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData();
+ if (doAllPartitionContainStats || css.size() < 2) {
+ LongColumnStatsData aggregateData = null;
+ long lowerBound = 0;
+ long higherBound = 0;
+ double densityAvgSum = 0.0;
+ NumDistinctValueEstimator ndvEstimator = null;
+ if (isNDVBitVectorSet) {
+ ndvEstimator = new NumDistinctValueEstimator(numBitVectors);
+ }
+ for (ColumnStatistics cs : css) {
+ ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
+ LongColumnStatsData newData = cso.getStatsData().getLongStats();
+ if (useDensityFunctionForNDVEstimation) {
+ lowerBound = Math.max(lowerBound, newData.getNumDVs());
+ higherBound += newData.getNumDVs();
+ densityAvgSum += (newData.getHighValue() - newData.getLowValue()) / newData.getNumDVs();
+ }
+ if (isNDVBitVectorSet) {
+ ndvEstimator.mergeEstimators(new NumDistinctValueEstimator(newData.getBitVectors(),
+ ndvEstimator.getnumBitVectors()));
+ }
+ if (aggregateData == null) {
+ aggregateData = newData.deepCopy();
+ } else {
+ aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue()));
+ aggregateData
+ .setHighValue(Math.max(aggregateData.getHighValue(), newData.getHighValue()));
+ aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
+ aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
+ }
+ }
+ if (isNDVBitVectorSet) {
+ // if all the ColumnStatisticsObjs contain bitvectors, we do not need to
+ // use uniform distribution assumption because we can merge bitvectors
+ // to get a good estimation.
+ aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
+ } else {
+ if (useDensityFunctionForNDVEstimation) {
+ // We have estimation, lowerbound and higherbound. We use estimation
+ // if it is between lowerbound and higherbound.
+ double densityAvg = densityAvgSum / partNames.size();
+ long estimation = (long) ((aggregateData.getHighValue() - aggregateData.getLowValue()) / densityAvg);
+ if (estimation < lowerBound) {
+ aggregateData.setNumDVs(lowerBound);
+ } else if (estimation > higherBound) {
+ aggregateData.setNumDVs(higherBound);
+ } else {
+ aggregateData.setNumDVs(estimation);
+ }
+ } else {
+ // Without useDensityFunctionForNDVEstimation, we just use the
+ // default one, which is the max of all the partitions and it is
+ // already done.
+ }
+ }
+ columnStatisticsData.setLongStats(aggregateData);
} else {
- ndvEstimator.mergeEstimators(new NumDistinctValueEstimator(newData.getBitVectors(),
- ndvEstimator.getnumBitVectors()));
- aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
- aggregateData.setBitVectors(ndvEstimator.serialize().toString());
+ // we need extrapolation
+ Map<String, Integer> indexMap = new HashMap<String, Integer>();
+ for (int index = 0; index < partNames.size(); index++) {
+ indexMap.put(partNames.get(index), index);
+ }
+ Map<String, Double> adjustedIndexMap = new HashMap<String, Double>();
+ Map<String, ColumnStatisticsData> adjustedStatsMap = new HashMap<String, ColumnStatisticsData>();
+ // while we scan the css, we also get the densityAvg, lowerbound and
+ // higerbound when useDensityFunctionForNDVEstimation is true.
+ double densityAvgSum = 0.0;
+ if (!isNDVBitVectorSet) {
+ // if not every partition uses bitvector for ndv, we just fall back to
+ // the traditional extrapolation methods.
+ for (ColumnStatistics cs : css) {
+ String partName = cs.getStatsDesc().getPartName();
+ ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
+ LongColumnStatsData newData = cso.getStatsData().getLongStats();
+ if (useDensityFunctionForNDVEstimation) {
+ densityAvgSum += (newData.getHighValue() - newData.getLowValue()) / newData.getNumDVs();
+ }
+ adjustedIndexMap.put(partName, (double) indexMap.get(partName));
+ adjustedStatsMap.put(partName, cso.getStatsData());
+ }
+ } else {
+ // we first merge all the adjacent bitvectors that we could merge and
+ // derive new partition names and index.
+ NumDistinctValueEstimator ndvEstimator = new NumDistinctValueEstimator(numBitVectors);
+ StringBuilder pseudoPartName = new StringBuilder();
+ double pseudoIndexSum = 0;
+ int length = 0;
+ int curIndex = -1;
+ LongColumnStatsData aggregateData = null;
+ for (ColumnStatistics cs : css) {
+ String partName = cs.getStatsDesc().getPartName();
+ ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
+ LongColumnStatsData newData = cso.getStatsData().getLongStats();
+ // newData.isSetBitVectors() should be true for sure because we
+ // already checked it before.
+ if (indexMap.get(partName) != curIndex) {
+ // There is bitvector, but it is not adjacent to the previous ones.
+ if (length > 0) {
+ // we have to set ndv
+ adjustedIndexMap.put(pseudoPartName.toString(), pseudoIndexSum / length);
+ aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
+ ColumnStatisticsData csd = new ColumnStatisticsData();
+ csd.setLongStats(aggregateData);
+ adjustedStatsMap.put(pseudoPartName.toString(), csd);
+ if (useDensityFunctionForNDVEstimation) {
+ densityAvgSum += (aggregateData.getHighValue() - aggregateData.getLowValue()) / aggregateData.getNumDVs();
+ }
+ // reset everything
+ pseudoPartName = new StringBuilder();
+ pseudoIndexSum = 0;
+ length = 0;
+ }
+ aggregateData = null;
+ }
+ curIndex = indexMap.get(partName);
+ pseudoPartName.append(partName);
+ pseudoIndexSum += curIndex;
+ length++;
+ curIndex++;
+ if (aggregateData == null) {
+ aggregateData = newData.deepCopy();
+ } else {
+ aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue()));
+ aggregateData.setHighValue(Math.max(aggregateData.getHighValue(),
+ newData.getHighValue()));
+ aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
+ }
+ ndvEstimator.mergeEstimators(new NumDistinctValueEstimator(newData.getBitVectors(),
+ ndvEstimator.getnumBitVectors()));
+ }
+ if (length > 0) {
+ // we have to set ndv
+ adjustedIndexMap.put(pseudoPartName.toString(), pseudoIndexSum / length);
+ aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
+ ColumnStatisticsData csd = new ColumnStatisticsData();
+ csd.setLongStats(aggregateData);
+ adjustedStatsMap.put(pseudoPartName.toString(), csd);
+ if (useDensityFunctionForNDVEstimation) {
+ densityAvgSum += (aggregateData.getHighValue() - aggregateData.getLowValue()) / aggregateData.getNumDVs();
+ }
+ }
+ }
+ extrapolate(columnStatisticsData, partNames.size(), css.size(), adjustedIndexMap,
+ adjustedStatsMap, densityAvgSum / adjustedStatsMap.size());
}
+ statsObj.setStatsData(columnStatisticsData);
+ return statsObj;
}
+
+ @Override
+ public void extrapolate(ColumnStatisticsData extrapolateData, int numParts,
+ int numPartsWithStats, Map<String, Double> adjustedIndexMap,
+ Map<String, ColumnStatisticsData> adjustedStatsMap, double densityAvg) {
+ int rightBorderInd = numParts;
+ LongColumnStatsData extrapolateLongData = new LongColumnStatsData();
+ Map<String, LongColumnStatsData> extractedAdjustedStatsMap = new HashMap<>();
+ for (Map.Entry<String, ColumnStatisticsData> entry : adjustedStatsMap.entrySet()) {
+ extractedAdjustedStatsMap.put(entry.getKey(), entry.getValue().getLongStats());
+ }
+ List<Map.Entry<String, LongColumnStatsData>> list = new LinkedList<Map.Entry<String, LongColumnStatsData>>(
+ extractedAdjustedStatsMap.entrySet());
+ // get the lowValue
+ Collections.sort(list, new Comparator<Map.Entry<String, LongColumnStatsData>>() {
+ public int compare(Map.Entry<String, LongColumnStatsData> o1,
+ Map.Entry<String, LongColumnStatsData> o2) {
+ return o1.getValue().getLowValue() < o2.getValue().getLowValue() ? -1 : 1;
+ }
+ });
+ double minInd = adjustedIndexMap.get(list.get(0).getKey());
+ double maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
+ long lowValue = 0;
+ long min = list.get(0).getValue().getLowValue();
+ long max = list.get(list.size() - 1).getValue().getLowValue();
+ if (minInd == maxInd) {
+ lowValue = min;
+ } else if (minInd < maxInd) {
+ // left border is the min
+ lowValue = (long) (max - (max - min) * maxInd / (maxInd - minInd));
+ } else {
+ // right border is the min
+ lowValue = (long) (max - (max - min) * (rightBorderInd - maxInd) / (minInd - maxInd));
+ }
+
+ // get the highValue
+ Collections.sort(list, new Comparator<Map.Entry<String, LongColumnStatsData>>() {
+ public int compare(Map.Entry<String, LongColumnStatsData> o1,
+ Map.Entry<String, LongColumnStatsData> o2) {
+ return o1.getValue().getHighValue() < o2.getValue().getHighValue() ? -1 : 1;
+ }
+ });
+ minInd = adjustedIndexMap.get(list.get(0).getKey());
+ maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
+ long highValue = 0;
+ min = list.get(0).getValue().getHighValue();
+ max = list.get(list.size() - 1).getValue().getHighValue();
+ if (minInd == maxInd) {
+ highValue = min;
+ } else if (minInd < maxInd) {
+ // right border is the max
+ highValue = (long) (min + (max - min) * (rightBorderInd - minInd) / (maxInd - minInd));
+ } else {
+ // left border is the max
+ highValue = (long) (min + (max - min) * minInd / (minInd - maxInd));
+ }
+
+ // get the #nulls
+ long numNulls = 0;
+ for (Map.Entry<String, LongColumnStatsData> entry : extractedAdjustedStatsMap.entrySet()) {
+ numNulls += entry.getValue().getNumNulls();
+ }
+ // we scale up sumNulls based on the number of partitions
+ numNulls = numNulls * numParts / numPartsWithStats;
+
+ // get the ndv
+ long ndv = 0;
+ Collections.sort(list, new Comparator<Map.Entry<String, LongColumnStatsData>>() {
+ public int compare(Map.Entry<String, LongColumnStatsData> o1,
+ Map.Entry<String, LongColumnStatsData> o2) {
+ return o1.getValue().getNumDVs() < o2.getValue().getNumDVs() ? -1 : 1;
+ }
+ });
+ long lowerBound = list.get(list.size() - 1).getValue().getNumDVs();
+ long higherBound = 0;
+ for (Map.Entry<String, LongColumnStatsData> entry : list) {
+ higherBound += entry.getValue().getNumDVs();
+ }
+ if (useDensityFunctionForNDVEstimation && densityAvg != 0.0) {
+ ndv = (long) ((highValue - lowValue) / densityAvg);
+ if (ndv < lowerBound) {
+ ndv = lowerBound;
+ } else if (ndv > higherBound) {
+ ndv = higherBound;
+ }
+ } else {
+ minInd = adjustedIndexMap.get(list.get(0).getKey());
+ maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
+ min = list.get(0).getValue().getNumDVs();
+ max = list.get(list.size() - 1).getValue().getNumDVs();
+ if (minInd == maxInd) {
+ ndv = min;
+ } else if (minInd < maxInd) {
+ // right border is the max
+ ndv = (long) (min + (max - min) * (rightBorderInd - minInd) / (maxInd - minInd));
+ } else {
+ // left border is the max
+ ndv = (long) (min + (max - min) * minInd / (minInd - maxInd));
+ }
+ }
+ extrapolateLongData.setLowValue(lowValue);
+ extrapolateLongData.setHighValue(highValue);
+ extrapolateLongData.setNumNulls(numNulls);
+ extrapolateLongData.setNumDVs(ndv);
+ extrapolateData.setLongStats(extrapolateLongData);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hive/blob/96862093/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/StringColumnStatsAggregator.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/StringColumnStatsAggregator.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/StringColumnStatsAggregator.java
index aeb6c39..2aa4046 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/StringColumnStatsAggregator.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/StringColumnStatsAggregator.java
@@ -19,26 +19,87 @@
package org.apache.hadoop.hive.metastore.hbase.stats;
+import java.util.List;
+
import org.apache.hadoop.hive.metastore.NumDistinctValueEstimator;
+import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
public class StringColumnStatsAggregator extends ColumnStatsAggregator {
@Override
- public void aggregate(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
- StringColumnStatsData aggregateData = aggregateColStats.getStatsData().getStringStats();
- StringColumnStatsData newData = newColStats.getStatsData().getStringStats();
- aggregateData.setMaxColLen(Math.max(aggregateData.getMaxColLen(), newData.getMaxColLen()));
- aggregateData.setAvgColLen(Math.max(aggregateData.getAvgColLen(), newData.getAvgColLen()));
- aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
- if (ndvEstimator == null || !newData.isSetBitVectors() || newData.getBitVectors().length() == 0) {
- aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
- } else {
- ndvEstimator.mergeEstimators(new NumDistinctValueEstimator(newData.getBitVectors(),
- ndvEstimator.getnumBitVectors()));
+ public ColumnStatisticsObj aggregate(String colName, List<String> partNames,
+ List<ColumnStatistics> css) throws MetaException {
+ ColumnStatisticsObj statsObj = null;
+
+ // check if all the ColumnStatisticsObjs contain stats and all the ndv are
+ // bitvectors. Only when both of the conditions are true, we merge bit
+ // vectors. Otherwise, just use the maximum function.
+ boolean doAllPartitionContainStats = partNames.size() == css.size();
+ boolean isNDVBitVectorSet = true;
+ String colType = null;
+ for (ColumnStatistics cs : css) {
+ if (cs.getStatsObjSize() != 1) {
+ throw new MetaException(
+ "The number of columns should be exactly one in aggrStats, but found "
+ + cs.getStatsObjSize());
+ }
+ ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
+ if (statsObj == null) {
+ colType = cso.getColType();
+ statsObj = ColumnStatsAggregatorFactory.newColumnStaticsObj(colName, colType, cso
+ .getStatsData().getSetField());
+ }
+ if (numBitVectors <= 0 || !cso.getStatsData().getStringStats().isSetBitVectors()
+ || cso.getStatsData().getStringStats().getBitVectors().length() == 0) {
+ isNDVBitVectorSet = false;
+ break;
+ }
+ }
+ ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData();
+ if (doAllPartitionContainStats && isNDVBitVectorSet) {
+ StringColumnStatsData aggregateData = null;
+ NumDistinctValueEstimator ndvEstimator = new NumDistinctValueEstimator(numBitVectors);
+ for (ColumnStatistics cs : css) {
+ ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
+ StringColumnStatsData newData = cso.getStatsData().getStringStats();
+ ndvEstimator.mergeEstimators(new NumDistinctValueEstimator(newData.getBitVectors(),
+ ndvEstimator.getnumBitVectors()));
+ if (aggregateData == null) {
+ aggregateData = newData.deepCopy();
+ } else {
+ aggregateData
+ .setMaxColLen(Math.max(aggregateData.getMaxColLen(), newData.getMaxColLen()));
+ aggregateData
+ .setAvgColLen(Math.max(aggregateData.getAvgColLen(), newData.getAvgColLen()));
+ aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
+ }
+ }
aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
- aggregateData.setBitVectors(ndvEstimator.serialize().toString());
+ columnStatisticsData.setStringStats(aggregateData);
+ } else {
+ StringColumnStatsData aggregateData = null;
+ for (ColumnStatistics cs : css) {
+ ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
+ StringColumnStatsData newData = cso.getStatsData().getStringStats();
+ if (aggregateData == null) {
+ aggregateData = newData.deepCopy();
+ } else {
+ aggregateData
+ .setMaxColLen(Math.max(aggregateData.getMaxColLen(), newData.getMaxColLen()));
+ aggregateData
+ .setAvgColLen(Math.max(aggregateData.getAvgColLen(), newData.getAvgColLen()));
+ aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
+ aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
+ }
+ }
+ columnStatisticsData.setStringStats(aggregateData);
}
+ statsObj.setStatsData(columnStatisticsData);
+ return statsObj;
}
+
}
http://git-wip-us.apache.org/repos/asf/hive/blob/96862093/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsCacheWithBitVector.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsCacheWithBitVector.java b/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsCacheWithBitVector.java
index 36c7984..e0c4094 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsCacheWithBitVector.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsCacheWithBitVector.java
@@ -156,10 +156,8 @@ public class TestHBaseAggregateStatsCacheWithBitVector {
Assert.assertEquals(-20.12, dcsd.getLowValue(), 0.01);
Assert.assertEquals(60, dcsd.getNumNulls());
Assert.assertEquals(5, dcsd.getNumDVs());
- Assert
- .assertEquals(
- "{0, 1, 4, 5, 7}{0, 1}{0, 1, 2, 4}{0, 1, 2, 4}{0, 1, 2}{0, 2}{0, 1, 3, 4}{0, 1, 2, 3, 4}{0, 1, 4}{0, 1, 3, 4, 6}{0, 2}{0, 1, 3, 8}{0, 2, 3}{0, 2}{0, 1, 9}{0, 1, 4}",
- dcsd.getBitVectors());
+ // we do not store the bitvector for the aggrStats.
+ // we can store that if it is necessary in the future.
}
};
[08/50] [abbrv] hive git commit: HIVE-13262: LLAP: Remove log levels
from DebugUtils (Prasanth Jayachandran reviewed by Sergey Shelukhin)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/3b6b56d7/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
index 29b51ec..f4cfa53 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
@@ -42,7 +42,6 @@ import org.apache.hadoop.hive.ql.io.orc.RecordReaderUtils;
import org.apache.orc.impl.StreamName;
import org.apache.orc.StripeInformation;
import org.apache.orc.impl.BufferChunk;
-import org.apache.hadoop.hive.ql.io.orc.RecordReaderUtils.ByteBufferAllocatorPool;
import org.apache.hadoop.hive.ql.io.orc.encoded.Reader.OrcEncodedColumnBatch;
import org.apache.hadoop.hive.ql.io.orc.encoded.Reader.PoolFactory;
import org.apache.orc.OrcProto;
@@ -103,8 +102,7 @@ class EncodedReaderImpl implements EncodedReader {
private final List<OrcProto.Type> types;
private final long rowIndexStride;
private final DataCache cache;
- private ByteBufferAllocatorPool pool;
- private boolean isDebugTracingEnabled;
+ private boolean isTracingEnabled;
public EncodedReaderImpl(Object fileKey, List<OrcProto.Type> types, CompressionCodec codec,
int bufferSize, long strideRate, DataCache cache, DataReader dataReader, PoolFactory pf)
@@ -209,8 +207,8 @@ class EncodedReaderImpl implements EncodedReader {
long offset = 0; // Stream offset in relation to the stripe.
// 1.1. Figure out which columns have a present stream
boolean[] hasNull = RecordReaderUtils.findPresentStreamsByColumn(streamList, types);
- if (isDebugTracingEnabled) {
- LOG.info("The following columns have PRESENT streams: " + arrayToString(hasNull));
+ if (isTracingEnabled) {
+ LOG.trace("The following columns have PRESENT streams: " + arrayToString(hasNull));
}
// We assume stream list is sorted by column and that non-data
@@ -230,8 +228,8 @@ class EncodedReaderImpl implements EncodedReader {
// We have a stream for included column, but in future it might have no data streams.
// It's more like "has at least one column included that has an index stream".
hasIndexOnlyCols = hasIndexOnlyCols | included[colIx];
- if (isDebugTracingEnabled) {
- LOG.info("Skipping stream: " + streamKind + " at " + offset + ", " + length);
+ if (isTracingEnabled) {
+ LOG.trace("Skipping stream: " + streamKind + " at " + offset + ", " + length);
}
offset += length;
continue;
@@ -244,8 +242,8 @@ class EncodedReaderImpl implements EncodedReader {
includedRgs = colRgs[colRgIx];
ctx = colCtxs[colRgIx] = new ColumnReadContext(
colIx, encodings.get(colIx), indexes[colIx]);
- if (isDebugTracingEnabled) {
- LOG.info("Creating context " + colRgIx + " for column " + colIx + ":" + ctx.toString());
+ if (isTracingEnabled) {
+ LOG.trace("Creating context " + colRgIx + " for column " + colIx + ":" + ctx.toString());
}
} else {
ctx = colCtxs[colRgIx];
@@ -254,14 +252,14 @@ class EncodedReaderImpl implements EncodedReader {
int indexIx = RecordReaderUtils.getIndexPosition(ctx.encoding.getKind(),
types.get(colIx).getKind(), streamKind, isCompressed, hasNull[colIx]);
ctx.addStream(offset, stream, indexIx);
- if (isDebugTracingEnabled) {
- LOG.info("Adding stream for column " + colIx + ": " + streamKind + " at " + offset
+ if (isTracingEnabled) {
+ LOG.trace("Adding stream for column " + colIx + ": " + streamKind + " at " + offset
+ ", " + length + ", index position " + indexIx);
}
if (includedRgs == null || RecordReaderUtils.isDictionary(streamKind, encodings.get(colIx))) {
RecordReaderUtils.addEntireStreamToRanges(offset, length, listToRead, true);
- if (isDebugTracingEnabled) {
- LOG.info("Will read whole stream " + streamKind + "; added to " + listToRead.getTail());
+ if (isTracingEnabled) {
+ LOG.trace("Will read whole stream " + streamKind + "; added to " + listToRead.getTail());
}
} else {
RecordReaderUtils.addRgFilteredStreamToRanges(stream, includedRgs,
@@ -287,15 +285,15 @@ class EncodedReaderImpl implements EncodedReader {
// 2. Now, read all of the ranges from cache or disk.
DiskRangeList.MutateHelper toRead = new DiskRangeList.MutateHelper(listToRead.get());
- if (isDebugTracingEnabled && LOG.isInfoEnabled()) {
- LOG.info("Resulting disk ranges to read (file " + fileKey + "): "
+ if (isTracingEnabled && LOG.isInfoEnabled()) {
+ LOG.trace("Resulting disk ranges to read (file " + fileKey + "): "
+ RecordReaderUtils.stringifyDiskRanges(toRead.next));
}
BooleanRef isAllInCache = new BooleanRef();
if (hasFileId) {
cache.getFileData(fileKey, toRead.next, stripeOffset, CC_FACTORY, isAllInCache);
- if (isDebugTracingEnabled && LOG.isInfoEnabled()) {
- LOG.info("Disk ranges after cache (file " + fileKey + ", base offset " + stripeOffset
+ if (isTracingEnabled && LOG.isInfoEnabled()) {
+ LOG.trace("Disk ranges after cache (file " + fileKey + ", base offset " + stripeOffset
+ "): " + RecordReaderUtils.stringifyDiskRanges(toRead.next));
}
}
@@ -322,8 +320,8 @@ class EncodedReaderImpl implements EncodedReader {
}
}
}
- if (isDebugTracingEnabled) {
- LOG.info("Disk ranges after pre-read (file " + fileKey + ", base offset "
+ if (isTracingEnabled) {
+ LOG.trace("Disk ranges after pre-read (file " + fileKey + ", base offset "
+ stripeOffset + "): " + RecordReaderUtils.stringifyDiskRanges(toRead.next));
}
iter = toRead.next; // Reset the iter to start.
@@ -354,8 +352,8 @@ class EncodedReaderImpl implements EncodedReader {
ColumnStreamData cb = null;
if (RecordReaderUtils.isDictionary(sctx.kind, ctx.encoding)) {
// This stream is for entire stripe and needed for every RG; uncompress once and reuse.
- if (isDebugTracingEnabled) {
- LOG.info("Getting stripe-level stream [" + sctx.kind + ", " + ctx.encoding + "] for"
+ if (isTracingEnabled) {
+ LOG.trace("Getting stripe-level stream [" + sctx.kind + ", " + ctx.encoding + "] for"
+ " column " + ctx.colIx + " RG " + rgIx + " at " + sctx.offset + ", " + sctx.length);
}
if (sctx.stripeLevelStream == null) {
@@ -411,8 +409,8 @@ class EncodedReaderImpl implements EncodedReader {
}
}
- if (isDebugTracingEnabled) {
- LOG.info("Disk ranges after preparing all the data "
+ if (isTracingEnabled) {
+ LOG.trace("Disk ranges after preparing all the data "
+ RecordReaderUtils.stringifyDiskRanges(toRead.next));
}
@@ -437,8 +435,8 @@ class EncodedReaderImpl implements EncodedReader {
int colIx, StreamContext sctx, long cOffset, long endCOffset, boolean isCompressed) {
ColumnStreamData cb = POOLS.csdPool.take();
cb.incRef();
- if (isDebugTracingEnabled) {
- LOG.info("Getting data for column "+ colIx + " " + (isLastRg ? "last " : "")
+ if (isTracingEnabled) {
+ LOG.trace("Getting data for column "+ colIx + " " + (isLastRg ? "last " : "")
+ "RG " + rgIx + " stream " + sctx.kind + " at " + sctx.offset + ", "
+ sctx.length + " index position " + sctx.streamIndexOffset + ": " +
(isCompressed ? "" : "un") + "compressed [" + cOffset + ", " + endCOffset + ")");
@@ -460,17 +458,14 @@ class EncodedReaderImpl implements EncodedReader {
}
@Override
- public void setDebugTracing(boolean isEnabled) {
- this.isDebugTracingEnabled = isEnabled;
+ public void setTracing(boolean isEnabled) {
+ this.isTracingEnabled = isEnabled;
}
@Override
public void close() throws IOException {
dataReader.close();
- if (pool != null) {
- pool.clear();
- }
}
/**
@@ -608,8 +603,8 @@ class EncodedReaderImpl implements EncodedReader {
// want to be, or just before. However, RGs can overlap due to encoding, so we may have
// to return to a previous block.
DiskRangeList current = findExactPosition(start, cOffset);
- if (isDebugTracingEnabled) {
- LOG.info("Starting read for [" + cOffset + "," + endCOffset + ") at " + current);
+ if (isTracingEnabled) {
+ LOG.trace("Starting read for [" + cOffset + "," + endCOffset + ") at " + current);
}
CacheChunk lastUncompressed = null;
@@ -648,8 +643,8 @@ class EncodedReaderImpl implements EncodedReader {
}
chunk.originalData = null;
- if (isDebugTracingEnabled) {
- LOG.info("Locking " + chunk.getBuffer() + " due to reuse (after decompression)");
+ if (isTracingEnabled) {
+ LOG.trace("Locking " + chunk.getBuffer() + " due to reuse (after decompression)");
}
cache.reuseBuffer(chunk.getBuffer());
}
@@ -691,22 +686,22 @@ class EncodedReaderImpl implements EncodedReader {
if (current instanceof CacheChunk) {
// 2a. This is a decoded compression buffer, add as is.
CacheChunk cc = (CacheChunk)current;
- if (isDebugTracingEnabled) {
- LOG.info("Locking " + cc.getBuffer() + " due to reuse");
+ if (isTracingEnabled) {
+ LOG.trace("Locking " + cc.getBuffer() + " due to reuse");
}
cache.reuseBuffer(cc.getBuffer());
columnStreamData.getCacheBuffers().add(cc.getBuffer());
currentOffset = cc.getEnd();
- if (isDebugTracingEnabled) {
- LOG.info("Adding an already-uncompressed buffer " + cc.getBuffer());
+ if (isTracingEnabled) {
+ LOG.trace("Adding an already-uncompressed buffer " + cc.getBuffer());
}
ponderReleaseInitialRefcount(unlockUntilCOffset, streamOffset, cc);
lastUncompressed = cc;
next = current.next;
} else if (current instanceof IncompleteCb) {
// 2b. This is a known incomplete CB caused by ORC CB end boundaries being estimates.
- if (isDebugTracingEnabled) {
- LOG.info("Cannot read " + current);
+ if (isTracingEnabled) {
+ LOG.trace("Cannot read " + current);
}
next = null;
currentOffset = -1;
@@ -739,8 +734,8 @@ class EncodedReaderImpl implements EncodedReader {
DiskRangeList next = null;
assert current instanceof CacheChunk;
lastUncompressed = (CacheChunk)current;
- if (isDebugTracingEnabled) {
- LOG.info("Locking " + lastUncompressed.getBuffer() + " due to reuse");
+ if (isTracingEnabled) {
+ LOG.trace("Locking " + lastUncompressed.getBuffer() + " due to reuse");
}
cache.reuseBuffer(lastUncompressed.getBuffer());
if (isFirst) {
@@ -749,8 +744,8 @@ class EncodedReaderImpl implements EncodedReader {
}
columnStreamData.getCacheBuffers().add(lastUncompressed.getBuffer());
currentOffset = lastUncompressed.getEnd();
- if (isDebugTracingEnabled) {
- LOG.info("Adding an uncompressed buffer " + lastUncompressed.getBuffer());
+ if (isTracingEnabled) {
+ LOG.trace("Adding an uncompressed buffer " + lastUncompressed.getBuffer());
}
ponderReleaseInitialRefcount(unlockUntilCOffset, streamOffset, lastUncompressed);
next = current.next;
@@ -770,7 +765,6 @@ class EncodedReaderImpl implements EncodedReader {
* to handle just for this case.
* We could avoid copy in non-zcr case and manage the buffer that was not allocated by our
* allocator. Uncompressed case is not mainline though so let's not complicate it.
- * @param qfCounters
*/
private DiskRangeList preReadUncompressedStream(long baseOffset,
DiskRangeList start, long streamOffset, long streamEnd) throws IOException {
@@ -780,8 +774,8 @@ class EncodedReaderImpl implements EncodedReader {
// 1. Find our bearings in the stream.
DiskRangeList current = findIntersectingPosition(start, streamOffset, streamEnd);
- if (isDebugTracingEnabled) {
- LOG.info("Starting pre-read for [" + streamOffset + "," + streamEnd + ") at " + current);
+ if (isTracingEnabled) {
+ LOG.trace("Starting pre-read for [" + streamOffset + "," + streamEnd + ") at " + current);
}
if (streamOffset > current.getOffset()) {
@@ -836,8 +830,8 @@ class EncodedReaderImpl implements EncodedReader {
current = current.split(partEnd);
wasSplit = true;
}
- if (isDebugTracingEnabled) {
- LOG.info("Processing uncompressed file data at ["
+ if (isTracingEnabled) {
+ LOG.trace("Processing uncompressed file data at ["
+ current.getOffset() + ", " + current.getEnd() + ")");
}
BufferChunk curBc = (BufferChunk)current;
@@ -1058,8 +1052,8 @@ class EncodedReaderImpl implements EncodedReader {
private void releaseInitialRefcount(CacheChunk cc, boolean isBacktracking) {
// This is the last RG for which this buffer will be used. Remove the initial refcount
- if (isDebugTracingEnabled) {
- LOG.info("Unlocking " + cc.getBuffer() + " for the fetching thread"
+ if (isTracingEnabled) {
+ LOG.trace("Unlocking " + cc.getBuffer() + " for the fetching thread"
+ (isBacktracking ? "; backtracking" : ""));
}
cache.releaseBuffer(cc.getBuffer());
@@ -1081,8 +1075,8 @@ class EncodedReaderImpl implements EncodedReader {
// Cache has found an old buffer for the key and put it into array instead of our new one.
CacheChunk replacedChunk = toDecompress.get(i);
MemoryBuffer replacementBuffer = targetBuffers[i];
- if (isDebugTracingEnabled) {
- LOG.info("Discarding data due to cache collision: " + replacedChunk.getBuffer()
+ if (isTracingEnabled) {
+ LOG.trace("Discarding data due to cache collision: " + replacedChunk.getBuffer()
+ " replaced with " + replacementBuffer);
}
assert replacedChunk.getBuffer() != replacementBuffer : i + " was not replaced in the results "
@@ -1133,7 +1127,6 @@ class EncodedReaderImpl implements EncodedReader {
* multiple ranges (usually, that would only happen with zcr).
* Adds stuff to cachedBuffers, toDecompress and toRelease (see below what each does).
* @param current BufferChunk where compression block starts.
- * @param ranges Iterator of all chunks, pointing at current.
* @param cacheBuffers The result buffer array to add pre-allocated target cache buffer.
* @param toDecompress The list of work to decompress - pairs of compressed buffers and the
* target buffers (same as the ones added to cacheBuffers).
@@ -1157,8 +1150,8 @@ class EncodedReaderImpl implements EncodedReader {
int consumedLength = chunkLength + OutStream.HEADER_SIZE;
long cbEndOffset = cbStartOffset + consumedLength;
boolean isUncompressed = ((b0 & 0x01) == 1);
- if (isDebugTracingEnabled) {
- LOG.info("Found CB at " + cbStartOffset + ", chunk length " + chunkLength + ", total "
+ if (isTracingEnabled) {
+ LOG.trace("Found CB at " + cbStartOffset + ", chunk length " + chunkLength + ", total "
+ consumedLength + ", " + (isUncompressed ? "not " : "") + "compressed");
}
if (compressed.remaining() >= chunkLength) {
@@ -1183,8 +1176,8 @@ class EncodedReaderImpl implements EncodedReader {
int remaining = chunkLength - compressed.remaining();
int originalPos = compressed.position();
copy.put(compressed);
- if (isDebugTracingEnabled) {
- LOG.info("Removing partial CB " + current + " from ranges after copying its contents");
+ if (isTracingEnabled) {
+ LOG.trace("Removing partial CB " + current + " from ranges after copying its contents");
}
DiskRangeList next = current.next;
current.removeSelf();
@@ -1223,8 +1216,8 @@ class EncodedReaderImpl implements EncodedReader {
DiskRangeList tmp = next;
next = next.hasContiguousNext() ? next.next : null;
if (next != null) {
- if (isDebugTracingEnabled) {
- LOG.info("Removing partial CB " + tmp + " from ranges after copying its contents");
+ if (isTracingEnabled) {
+ LOG.trace("Removing partial CB " + tmp + " from ranges after copying its contents");
}
tmp.removeSelf();
} else {
@@ -1237,8 +1230,8 @@ class EncodedReaderImpl implements EncodedReader {
private void addIncompleteCompressionBuffer(
long cbStartOffset, DiskRangeList target, int extraChunkCount) {
IncompleteCb icb = new IncompleteCb(cbStartOffset, target.getEnd());
- if (isDebugTracingEnabled) {
- LOG.info("Replacing " + target + " (and " + extraChunkCount + " previous chunks) with "
+ if (isTracingEnabled) {
+ LOG.trace("Replacing " + target + " (and " + extraChunkCount + " previous chunks) with "
+ icb + " in the buffers");
}
target.replaceSelfWith(icb);
@@ -1250,9 +1243,7 @@ class EncodedReaderImpl implements EncodedReader {
* @param isUncompressed Whether the data in the block is uncompressed.
* @param cbStartOffset Compressed start offset of the fCB.
* @param cbEndOffset Compressed end offset of the fCB.
- * @param lastRange The buffer from which the last (or all) bytes of fCB come.
* @param lastChunkLength The number of compressed bytes consumed from last *chunk* into fullCompressionBlock.
- * @param ranges The iterator of all compressed ranges for the stream, pointing at lastRange.
* @param lastChunk
* @param toDecompress See addOneCompressionBuffer.
* @param cacheBuffers See addOneCompressionBuffer.
@@ -1271,20 +1262,20 @@ class EncodedReaderImpl implements EncodedReader {
fullCompressionBlock, futureAlloc, cacheBuffers.size() - 1);
toDecompress.add(cc);
// Adjust the compression block position.
- if (isDebugTracingEnabled) {
- LOG.info("Adjusting " + lastChunk + " to consume " + lastChunkLength + " compressed bytes");
+ if (isTracingEnabled) {
+ LOG.trace("Adjusting " + lastChunk + " to consume " + lastChunkLength + " compressed bytes");
}
lastChunk.getChunk().position(lastChunk.getChunk().position() + lastChunkLength);
// Finally, put it in the ranges list for future use (if shared between RGs).
// Before anyone else accesses it, it would have been allocated and decompressed locally.
if (lastChunk.getChunk().remaining() <= 0) {
- if (isDebugTracingEnabled) {
- LOG.info("Replacing " + lastChunk + " with " + cc + " in the buffers");
+ if (isTracingEnabled) {
+ LOG.trace("Replacing " + lastChunk + " with " + cc + " in the buffers");
}
lastChunk.replaceSelfWith(cc);
} else {
- if (isDebugTracingEnabled) {
- LOG.info("Adding " + cc + " before " + lastChunk + " in the buffers");
+ if (isTracingEnabled) {
+ LOG.trace("Adding " + cc + " before " + lastChunk + " in the buffers");
}
lastChunk.insertPartBefore(cc);
}
[43/50] [abbrv] hive git commit: HIVE-13303 : spill to YARN
directories, not tmp, when available (Sergey Shelukhin, reviewed by Gopal V)
Posted by jd...@apache.org.
HIVE-13303 : spill to YARN directories, not tmp, when available (Sergey Shelukhin, reviewed by Gopal V)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/20a8192a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/20a8192a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/20a8192a
Branch: refs/heads/llap
Commit: 20a8192a2b8f36da5ef2d5d61d77de1e70188b1d
Parents: 56b6459
Author: Sergey Shelukhin <se...@apache.org>
Authored: Tue Mar 29 18:57:27 2016 -0700
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Tue Mar 29 18:57:27 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/common/FileUtils.java | 54 ++++++++++++++++++++
.../hadoop/hive/llap/io/api/LlapProxy.java | 2 +
.../org/apache/hadoop/hive/llap/LlapUtil.java | 26 ++++++++++
.../hive/llap/daemon/impl/LlapDaemon.java | 6 +--
.../persistence/HybridHashTableContainer.java | 40 ++++++++++-----
.../ql/exec/persistence/KeyValueContainer.java | 25 +++++----
.../ql/exec/persistence/ObjectContainer.java | 24 ++++-----
.../hive/ql/exec/persistence/RowContainer.java | 34 ++++++------
.../hadoop/hive/ql/exec/tez/DagUtils.java | 1 +
.../mapjoin/VectorMapJoinRowBytesContainer.java | 24 ++++-----
.../ql/exec/persistence/TestHashPartition.java | 3 +-
.../TestVectorMapJoinRowBytesContainer.java | 3 +-
12 files changed, 169 insertions(+), 73 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/20a8192a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
index 8c9bd3d..51340d8 100644
--- a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
+++ b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
@@ -27,6 +27,7 @@ import java.security.AccessControlException;
import java.security.PrivilegedExceptionAction;
import java.util.BitSet;
import java.util.List;
+import java.util.Random;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
@@ -43,6 +44,7 @@ import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.hive.shims.Utils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.util.StringUtils;
import org.apache.hive.common.util.ShutdownHookManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -53,6 +55,7 @@ import org.slf4j.LoggerFactory;
*/
public final class FileUtils {
private static final Logger LOG = LoggerFactory.getLogger(FileUtils.class.getName());
+ private static final Random random = new Random();
public static final PathFilter HIDDEN_FILES_PATH_FILTER = new PathFilter() {
@Override
@@ -827,6 +830,57 @@ public final class FileUtils {
return tmpFile;
}
+ public static File createLocalDirsTempFile(String localDirList, String prefix, String suffix,
+ boolean isDirectory) throws IOException {
+ if (localDirList == null || localDirList.isEmpty()) {
+ return createFileInTmp(prefix, suffix, "Local directories not specified", isDirectory);
+ }
+ String[] localDirs = StringUtils.getTrimmedStrings(localDirList);
+ if (localDirs.length == 0) {
+ return createFileInTmp(prefix, suffix, "Local directories not specified", isDirectory);
+ }
+ // TODO: we could stagger these to threads by ID, but that can also lead to bad effects.
+ String path = localDirs[random.nextInt(localDirs.length)];
+ if (path == null || path.isEmpty()) {
+ return createFileInTmp(prefix, suffix, "Empty path for one of the local dirs", isDirectory);
+ }
+ File targetDir = new File(path);
+ if (!targetDir.exists() && !targetDir.mkdirs()) {
+ return createFileInTmp(prefix, suffix, "Cannot access or create " + targetDir, isDirectory);
+ }
+ try {
+ File file = File.createTempFile(prefix, suffix, targetDir);
+ if (isDirectory && (!file.delete() || !file.mkdirs())) {
+ // TODO: or we could just generate a name ourselves and not do this?
+ return createFileInTmp(prefix, suffix,
+ "Cannot recreate " + file + " as directory", isDirectory);
+ }
+ file.deleteOnExit();
+ return file;
+ } catch (IOException ex) {
+ LOG.error("Error creating a file in " + targetDir, ex);
+ return createFileInTmp(prefix, suffix, "Cannot create a file in " + targetDir, isDirectory);
+ }
+ }
+
+ private static File createFileInTmp(String prefix, String suffix,
+ String reason, boolean isDirectory) throws IOException {
+ File file = File.createTempFile(prefix, suffix);
+ if (isDirectory && (!file.delete() || !file.mkdirs())) {
+ // TODO: or we could just generate a name ourselves and not do this?
+ throw new IOException("Cannot recreate " + file + " as directory");
+ }
+ file.deleteOnExit();
+ LOG.info(reason + "; created a tmp file: " + file.getAbsolutePath());
+ return file;
+ }
+
+ public static File createLocalDirsTempFile(Configuration conf, String prefix, String suffix,
+ boolean isDirectory) throws IOException {
+ return createLocalDirsTempFile(
+ conf.get("yarn.nodemanager.local-dirs"), prefix, suffix, isDirectory);
+ }
+
/**
* delete a temporary file and remove it from delete-on-exit hook.
*/
http://git-wip-us.apache.org/repos/asf/hive/blob/20a8192a/llap-client/src/java/org/apache/hadoop/hive/llap/io/api/LlapProxy.java
----------------------------------------------------------------------
diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/io/api/LlapProxy.java b/llap-client/src/java/org/apache/hadoop/hive/llap/io/api/LlapProxy.java
index 6359bab..424769f 100644
--- a/llap-client/src/java/org/apache/hadoop/hive/llap/io/api/LlapProxy.java
+++ b/llap-client/src/java/org/apache/hadoop/hive/llap/io/api/LlapProxy.java
@@ -21,6 +21,8 @@ import java.io.IOException;
import java.lang.reflect.Constructor;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.llap.security.LlapTokenProvider;
http://git-wip-us.apache.org/repos/asf/hive/blob/20a8192a/llap-common/src/java/org/apache/hadoop/hive/llap/LlapUtil.java
----------------------------------------------------------------------
diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/LlapUtil.java b/llap-common/src/java/org/apache/hadoop/hive/llap/LlapUtil.java
new file mode 100644
index 0000000..ce03de0
--- /dev/null
+++ b/llap-common/src/java/org/apache/hadoop/hive/llap/LlapUtil.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.llap;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+
+public class LlapUtil {
+ public static String getDaemonLocalDirList(Configuration conf) {
+ String localDirList = HiveConf.getVar(conf, ConfVars.LLAP_DAEMON_WORK_DIRS);
+ if (localDirList != null && !localDirList.isEmpty()) return localDirList;
+ return conf.get("yarn.nodemanager.local-dirs");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/20a8192a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
index 22d7eec..c8734a5 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
@@ -30,6 +30,7 @@ import javax.management.ObjectName;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.llap.LlapUtil;
import org.apache.hadoop.hive.llap.configuration.LlapDaemonConfiguration;
import org.apache.hadoop.hive.llap.daemon.ContainerRunner;
import org.apache.hadoop.hive.llap.daemon.QueryFailedHandler;
@@ -333,10 +334,7 @@ public class LlapDaemon extends CompositeService implements ContainerRunner, Lla
LlapDaemonConfiguration daemonConf = new LlapDaemonConfiguration();
int numExecutors = HiveConf.getIntVar(daemonConf, ConfVars.LLAP_DAEMON_NUM_EXECUTORS);
- String localDirList = HiveConf.getVar(daemonConf, ConfVars.LLAP_DAEMON_WORK_DIRS);
- if (localDirList == null || localDirList.isEmpty()) {
- localDirList = daemonConf.get("yarn.nodemanager.local-dirs");
- }
+ String localDirList = LlapUtil.getDaemonLocalDirList(daemonConf);
String[] localDirs = (localDirList == null || localDirList.isEmpty()) ?
new String[0] : StringUtils.getTrimmedStrings(localDirList);
int rpcPort = HiveConf.getIntVar(daemonConf, ConfVars.LLAP_DAEMON_RPC_PORT);
http://git-wip-us.apache.org/repos/asf/hive/blob/20a8192a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java
index f6471db..f5da5a4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java
@@ -19,6 +19,8 @@
package org.apache.hadoop.hive.ql.exec.persistence;
+import java.io.File;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.ObjectOutputStream;
@@ -30,6 +32,7 @@ import java.util.Collections;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator;
import org.apache.hadoop.hive.ql.exec.JoinUtil;
@@ -110,6 +113,8 @@ public class HybridHashTableContainer
private final List<Object> EMPTY_LIST = new ArrayList<Object>(0);
+ private final String spillLocalDirs;
+
/**
* This class encapsulates the triplet together since they are closely related to each other
* The triplet: hashmap (either in memory or on disk), small table container, big table container
@@ -127,12 +132,13 @@ public class HybridHashTableContainer
float loadFactor; // Same as above
int wbSize; // Same as above
int rowsOnDisk; // How many rows saved to the on-disk hashmap (if on disk)
+ private final String spillLocalDirs;
/* It may happen that there's not enough memory to instantiate a hashmap for the partition.
* In that case, we don't create the hashmap, but pretend the hashmap is directly "spilled".
*/
public HashPartition(int initialCapacity, float loadFactor, int wbSize, long maxProbeSize,
- boolean createHashMap) {
+ boolean createHashMap, String spillLocalDirs) {
if (createHashMap) {
// Probe space should be at least equal to the size of our designated wbSize
maxProbeSize = Math.max(maxProbeSize, wbSize);
@@ -141,6 +147,7 @@ public class HybridHashTableContainer
hashMapSpilledOnCreation = true;
hashMapOnDisk = true;
}
+ this.spillLocalDirs = spillLocalDirs;
this.initialCapacity = initialCapacity;
this.loadFactor = loadFactor;
this.wbSize = wbSize;
@@ -187,7 +194,7 @@ public class HybridHashTableContainer
/* Get the small table key/value container */
public KeyValueContainer getSidefileKVContainer() {
if (sidefileKVContainer == null) {
- sidefileKVContainer = new KeyValueContainer();
+ sidefileKVContainer = new KeyValueContainer(spillLocalDirs);
}
return sidefileKVContainer;
}
@@ -195,7 +202,7 @@ public class HybridHashTableContainer
/* Get the big table row container */
public ObjectContainer getMatchfileObjContainer() {
if (matchfileObjContainer == null) {
- matchfileObjContainer = new ObjectContainer();
+ matchfileObjContainer = new ObjectContainer(spillLocalDirs);
}
return matchfileObjContainer;
}
@@ -203,7 +210,7 @@ public class HybridHashTableContainer
/* Get the big table row bytes container for native vector map join */
public VectorMapJoinRowBytesContainer getMatchfileRowBytesContainer() {
if (matchfileRowBytesContainer == null) {
- matchfileRowBytesContainer = new VectorMapJoinRowBytesContainer();
+ matchfileRowBytesContainer = new VectorMapJoinRowBytesContainer(spillLocalDirs);
}
return matchfileRowBytesContainer;
}
@@ -267,12 +274,14 @@ public class HybridHashTableContainer
HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEHASHTABLEWBSIZE),
HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEHYBRIDGRACEHASHJOINMINNUMPARTITIONS),
HiveConf.getFloatVar(hconf, HiveConf.ConfVars.HIVEMAPJOINOPTIMIZEDTABLEPROBEPERCENT),
- estimatedTableSize, keyCount, memoryAvailable, nwayConf);
+ estimatedTableSize, keyCount, memoryAvailable, nwayConf,
+ RowContainer.getLocalDirsForSpillFiles(hconf));
}
private HybridHashTableContainer(float keyCountAdj, int threshold, float loadFactor,
int memCheckFreq, int minWbSize, int maxWbSize, int minNumParts, float probePercent,
- long estimatedTableSize, long keyCount, long memoryAvailable, HybridHashTableConf nwayConf)
+ long estimatedTableSize, long keyCount, long memoryAvailable, HybridHashTableConf nwayConf,
+ String spillLocalDirs)
throws SerDeException, IOException {
directWriteHelper = new MapJoinBytesTableContainer.DirectKeyValueWriter();
@@ -282,6 +291,7 @@ public class HybridHashTableContainer
memoryThreshold = memoryAvailable;
tableRowSize = estimatedTableSize / (keyCount != 0 ? keyCount : 1);
memoryCheckFrequency = memCheckFreq;
+ this.spillLocalDirs = spillLocalDirs;
this.nwayConf = nwayConf;
int writeBufferSize;
@@ -343,21 +353,22 @@ public class HybridHashTableContainer
nwayConf.getLoadedContainerList().size() == 0) { // n-way join, first (biggest) small table
if (i == 0) { // We unconditionally create a hashmap for the first hash partition
hashPartitions[i] = new HashPartition(initialCapacity, loadFactor, writeBufferSize,
- maxCapacity, true);
+ maxCapacity, true, spillLocalDirs);
} else {
// To check whether we have enough memory to allocate for another hash partition,
// we need to get the size of the first hash partition to get an idea.
hashPartitions[i] = new HashPartition(initialCapacity, loadFactor, writeBufferSize,
- maxCapacity, memoryUsed + hashPartitions[0].hashMap.memorySize() < memoryThreshold);
+ maxCapacity, memoryUsed + hashPartitions[0].hashMap.memorySize() < memoryThreshold,
+ spillLocalDirs);
}
} else { // n-way join, all later small tables
// For all later small tables, follow the same pattern of the previously loaded tables.
if (this.nwayConf.doSpillOnCreation(i)) {
hashPartitions[i] = new HashPartition(initialCapacity, loadFactor, writeBufferSize,
- maxCapacity, false);
+ maxCapacity, false, spillLocalDirs);
} else {
hashPartitions[i] = new HashPartition(initialCapacity, loadFactor, writeBufferSize,
- maxCapacity, true);
+ maxCapacity, true, spillLocalDirs);
}
}
@@ -542,8 +553,9 @@ public class HybridHashTableContainer
HashPartition partition = hashPartitions[partitionId];
int inMemRowCount = partition.hashMap.getNumValues();
- Path path = Files.createTempFile("partition-" + partitionId + "-", null);
- OutputStream outputStream = Files.newOutputStream(path);
+ File file = FileUtils.createLocalDirsTempFile(
+ spillLocalDirs, "partition-" + partitionId + "-", null, false);
+ OutputStream outputStream = new FileOutputStream(file, false);
com.esotericsoftware.kryo.io.Output output =
new com.esotericsoftware.kryo.io.Output(outputStream);
@@ -556,11 +568,11 @@ public class HybridHashTableContainer
SerializationUtilities.releaseKryo(kryo);
}
- partition.hashMapLocalPath = path;
+ partition.hashMapLocalPath = file.toPath();
partition.hashMapOnDisk = true;
LOG.info("Spilling hash partition " + partitionId + " (Rows: " + inMemRowCount +
- ", Mem size: " + partition.hashMap.memorySize() + "): " + path);
+ ", Mem size: " + partition.hashMap.memorySize() + "): " + file);
LOG.info("Memory usage before spilling: " + memoryUsed);
long memFreed = partition.hashMap.memorySize();
http://git-wip-us.apache.org/repos/asf/hive/blob/20a8192a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/KeyValueContainer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/KeyValueContainer.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/KeyValueContainer.java
index d403c58..e2b22d3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/KeyValueContainer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/KeyValueContainer.java
@@ -24,6 +24,7 @@ import com.google.common.base.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.common.ObjectPair;
import org.apache.hadoop.hive.ql.io.HiveKey;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -52,36 +53,34 @@ public class KeyValueContainer {
private int readCursor = 0; // cursor during reading
private int rowsOnDisk = 0; // total number of pairs in output
- private File parentFile;
+ private File parentDir;
private File tmpFile;
private Input input;
private Output output;
- public KeyValueContainer() {
+ public KeyValueContainer(String spillLocalDirs) {
readBuffer = new ObjectPair[IN_MEMORY_NUM_ROWS];
for (int i = 0; i < IN_MEMORY_NUM_ROWS; i++) {
readBuffer[i] = new ObjectPair<HiveKey, BytesWritable>();
}
try {
- setupOutput();
+ setupOutput(spillLocalDirs);
} catch (IOException | HiveException e) {
throw new RuntimeException("Failed to create temporary output file on disk", e);
}
}
- private void setupOutput() throws IOException, HiveException {
+ private void setupOutput(String spillLocalDirs) throws IOException, HiveException {
FileOutputStream fos = null;
try {
- if (parentFile == null) {
- parentFile = File.createTempFile("key-value-container", "");
- if (parentFile.delete() && parentFile.mkdir()) {
- parentFile.deleteOnExit();
- }
+ if (parentDir == null) {
+ parentDir = FileUtils.createLocalDirsTempFile(spillLocalDirs, "key-value-container", "", true);
+ parentDir.deleteOnExit();
}
if (tmpFile == null || input != null) {
- tmpFile = File.createTempFile("KeyValueContainer", ".tmp", parentFile);
+ tmpFile = File.createTempFile("KeyValueContainer", ".tmp", parentDir);
LOG.info("KeyValueContainer created temp file " + tmpFile.getAbsolutePath());
tmpFile.deleteOnExit();
}
@@ -131,7 +130,7 @@ public class KeyValueContainer {
readCursor = rowsInReadBuffer = rowsOnDisk = 0;
readBufferUsed = false;
- if (parentFile != null) {
+ if (parentDir != null) {
if (input != null) {
try {
input.close();
@@ -147,10 +146,10 @@ public class KeyValueContainer {
output = null;
}
try {
- FileUtil.fullyDelete(parentFile);
+ FileUtil.fullyDelete(parentDir);
} catch (Throwable ignored) {
}
- parentFile = null;
+ parentDir = null;
tmpFile = null;
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/20a8192a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/ObjectContainer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/ObjectContainer.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/ObjectContainer.java
index a976de0..ee9da23 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/ObjectContainer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/ObjectContainer.java
@@ -23,6 +23,7 @@ import java.io.FileOutputStream;
import java.io.IOException;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.ql.exec.SerializationUtilities;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.slf4j.Logger;
@@ -53,36 +54,33 @@ public class ObjectContainer<ROW> {
private int readCursor = 0; // cursor during reading
private int rowsOnDisk = 0; // total number of objects in output
- private File parentFile;
+ private File parentDir;
private File tmpFile;
private Input input;
private Output output;
- public ObjectContainer() {
+ public ObjectContainer(String spillLocalDirs) {
readBuffer = (ROW[]) new Object[IN_MEMORY_NUM_ROWS];
for (int i = 0; i < IN_MEMORY_NUM_ROWS; i++) {
readBuffer[i] = (ROW) new Object();
}
try {
- setupOutput();
+ setupOutput(spillLocalDirs);
} catch (IOException | HiveException e) {
throw new RuntimeException("Failed to create temporary output file on disk", e);
}
}
- private void setupOutput() throws IOException, HiveException {
+ private void setupOutput(String spillLocalDirs) throws IOException, HiveException {
FileOutputStream fos = null;
try {
- if (parentFile == null) {
- parentFile = File.createTempFile("object-container", "");
- if (parentFile.delete() && parentFile.mkdir()) {
- parentFile.deleteOnExit();
- }
+ if (parentDir == null) {
+ parentDir = FileUtils.createLocalDirsTempFile(spillLocalDirs, "object-container", "", true);
}
if (tmpFile == null || input != null) {
- tmpFile = File.createTempFile("ObjectContainer", ".tmp", parentFile);
+ tmpFile = File.createTempFile("ObjectContainer", ".tmp", parentDir);
LOG.info("ObjectContainer created temp file " + tmpFile.getAbsolutePath());
tmpFile.deleteOnExit();
}
@@ -112,7 +110,7 @@ public class ObjectContainer<ROW> {
readCursor = rowsInReadBuffer = rowsOnDisk = 0;
readBufferUsed = false;
- if (parentFile != null) {
+ if (parentDir != null) {
if (input != null) {
try {
input.close();
@@ -128,10 +126,10 @@ public class ObjectContainer<ROW> {
output = null;
}
try {
- FileUtil.fullyDelete(parentFile);
+ FileUtil.fullyDelete(parentDir);
} catch (Throwable ignored) {
}
- parentFile = null;
+ parentDir = null;
tmpFile = null;
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/20a8192a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java
index 358f692..893d265 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java
@@ -31,6 +31,9 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.FileUtils;
+import org.apache.hadoop.hive.llap.LlapUtil;
+import org.apache.hadoop.hive.llap.io.api.LlapProxy;
import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
@@ -91,7 +94,7 @@ public class RowContainer<ROW extends List<Object>>
private long size; // total # of elements in the RowContainer
private File tmpFile; // temporary file holding the spilled blocks
Path tempOutPath = null;
- private File parentFile;
+ private File parentDir;
private int itrCursor; // iterator cursor in the currBlock
private int readBlockSize; // size of current read block
private int addCursor; // append cursor in the lastBlock
@@ -112,6 +115,8 @@ public class RowContainer<ROW extends List<Object>>
InputSplit[] inputSplits = null;
private ROW dummyRow = null;
private final Reporter reporter;
+ private final String spillFileDirs;
+
Writable val = null; // cached to use serialize data
@@ -130,6 +135,7 @@ public class RowContainer<ROW extends List<Object>>
this.size = 0;
this.itrCursor = 0;
this.addCursor = 0;
+ this.spillFileDirs = getLocalDirsForSpillFiles(jc);
this.numFlushedBlocks = 0;
this.tmpFile = null;
this.currentWriteBlock = (ROW[]) new ArrayList[blockSize];
@@ -145,6 +151,11 @@ public class RowContainer<ROW extends List<Object>>
}
}
+ public static String getLocalDirsForSpillFiles(Configuration conf) {
+ return LlapProxy.isDaemon()
+ ? LlapUtil.getDaemonLocalDirList(conf) : conf.get("yarn.nodemanager.local-dirs");
+ }
+
private JobConf getLocalFSJobConfClone(Configuration jc) {
if (this.jobCloneUsingLocalFs == null) {
this.jobCloneUsingLocalFs = new JobConf(jc);
@@ -220,7 +231,7 @@ public class RowContainer<ROW extends List<Object>>
}
localJc.set(FileInputFormat.INPUT_DIR,
- org.apache.hadoop.util.StringUtils.escapeString(parentFile.getAbsolutePath()));
+ org.apache.hadoop.util.StringUtils.escapeString(parentDir.getAbsolutePath()));
inputSplits = inputFormat.getSplits(localJc, 1);
actualSplitNum = inputSplits.length;
}
@@ -289,7 +300,7 @@ public class RowContainer<ROW extends List<Object>>
}
private final ArrayList<Object> row = new ArrayList<Object>(2);
-
+
private void spillBlock(ROW[] block, int length) throws HiveException {
try {
if (tmpFile == null) {
@@ -445,8 +456,8 @@ public class RowContainer<ROW extends List<Object>>
rw = null;
rr = null;
tmpFile = null;
- deleteLocalFile(parentFile, true);
- parentFile = null;
+ deleteLocalFile(parentDir, true);
+ parentDir = null;
}
}
@@ -518,21 +529,14 @@ public class RowContainer<ROW extends List<Object>>
suffix = "." + this.keyObject.toString() + suffix;
}
- while (true) {
- parentFile = File.createTempFile("hive-rowcontainer", "");
- boolean success = parentFile.delete() && parentFile.mkdir();
- if (success) {
- break;
- }
- LOG.debug("retry creating tmp row-container directory...");
- }
+ parentDir = FileUtils.createLocalDirsTempFile(spillFileDirs, "hive-rowcontainer", "", true);
- tmpFile = File.createTempFile("RowContainer", suffix, parentFile);
+ tmpFile = File.createTempFile("RowContainer", suffix, parentDir);
LOG.info("RowContainer created temp file " + tmpFile.getAbsolutePath());
// Delete the temp file if the JVM terminate normally through Hadoop job
// kill command.
// Caveat: it won't be deleted if JVM is killed by 'kill -9'.
- parentFile.deleteOnExit();
+ parentDir.deleteOnExit();
tmpFile.deleteOnExit();
// rFile = new RandomAccessFile(tmpFile, "rw");
http://git-wip-us.apache.org/repos/asf/hive/blob/20a8192a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
index 79da860..8aca779 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
@@ -577,6 +577,7 @@ public class DagUtils {
}
}
+ // TODO# HERE?
if (mapWork instanceof MergeFileWork) {
Path outputPath = ((MergeFileWork) mapWork).getOutputDir();
// prepare the tmp output directory. The output tmp directory should
http://git-wip-us.apache.org/repos/asf/hive/blob/20a8192a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinRowBytesContainer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinRowBytesContainer.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinRowBytesContainer.java
index 4c539d8..fa96ae9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinRowBytesContainer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinRowBytesContainer.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.exec.vector.mapjoin;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.serde2.ByteStream.Output;
import java.io.File;
@@ -34,7 +35,7 @@ public class VectorMapJoinRowBytesContainer {
private static final Logger LOG = LoggerFactory.getLogger(VectorMapJoinRowBytesContainer.class);
- private File parentFile;
+ private File parentDir;
private File tmpFile;
// We buffer in a org.apache.hadoop.hive.serde2.ByteStream.Output since that is what
@@ -72,7 +73,9 @@ public class VectorMapJoinRowBytesContainer {
private FileInputStream fileInputStream;
- public VectorMapJoinRowBytesContainer() {
+ private final String spillLocalDirs;
+
+ public VectorMapJoinRowBytesContainer(String spillLocalDirs) {
output = new Output();
readBuffer = new byte[INPUT_SIZE];
readNextBytes = new byte[MAX_READS][];
@@ -81,16 +84,13 @@ public class VectorMapJoinRowBytesContainer {
isOpen = false;
totalWriteLength = 0;
totalReadLength = 0;
+ this.spillLocalDirs = spillLocalDirs;
}
private void setupOutputFileStreams() throws IOException {
-
- parentFile = File.createTempFile("bytes-container", "");
- if (parentFile.delete() && parentFile.mkdir()) {
- parentFile.deleteOnExit();
- }
-
- tmpFile = File.createTempFile("BytesContainer", ".tmp", parentFile);
+ parentDir = FileUtils.createLocalDirsTempFile(spillLocalDirs, "bytes-container", "", true);
+ parentDir.deleteOnExit();
+ tmpFile = File.createTempFile("BytesContainer", ".tmp", parentDir);
LOG.debug("BytesContainer created temp file " + tmpFile.getAbsolutePath());
tmpFile.deleteOnExit();
@@ -306,13 +306,13 @@ public class VectorMapJoinRowBytesContainer {
fileOutputStream = null;
}
- if (parentFile != null) {
+ if (parentDir != null) {
try {
- FileUtil.fullyDelete(parentFile);
+ FileUtil.fullyDelete(parentDir);
} catch (Throwable ignored) {
}
}
- parentFile = null;
+ parentDir = null;
tmpFile = null;
isOpen = false;
totalWriteLength = 0;
http://git-wip-us.apache.org/repos/asf/hive/blob/20a8192a/ql/src/test/org/apache/hadoop/hive/ql/exec/persistence/TestHashPartition.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/persistence/TestHashPartition.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/persistence/TestHashPartition.java
index a6e52bd..efabd2b 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/persistence/TestHashPartition.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/persistence/TestHashPartition.java
@@ -24,6 +24,7 @@ public class TestHashPartition {
@Test
public void testHashPartition() throws Exception {
- HashPartition hashPartition = new HashPartition(1024, (float) 0.75, 524288, 1, true);
+ // TODO: wtf?
+ HashPartition hashPartition = new HashPartition(1024, (float) 0.75, 524288, 1, true, null);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/20a8192a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinRowBytesContainer.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinRowBytesContainer.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinRowBytesContainer.java
index 3c3aacd..afe4e70 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinRowBytesContainer.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinRowBytesContainer.java
@@ -31,7 +31,8 @@ public class TestVectorMapJoinRowBytesContainer {
public void doFillReplay(Random random, int maxCount) throws Exception {
RandomByteArrayStream randomByteArrayStream = new RandomByteArrayStream(random);
- VectorMapJoinRowBytesContainer vectorMapJoinRowBytesContainer = new VectorMapJoinRowBytesContainer();
+ VectorMapJoinRowBytesContainer vectorMapJoinRowBytesContainer =
+ new VectorMapJoinRowBytesContainer(null);
int count = Math.min(maxCount, random.nextInt(500));
for (int i = 0; i < count; i++) {
[10/50] [abbrv] hive git commit: HIVE-9499 :
hive.limit.query.max.table.partition makes queries fail on non-partitioned
tables (Navis via Ashutosh Chauhan)
Posted by jd...@apache.org.
HIVE-9499 : hive.limit.query.max.table.partition makes queries fail on non-partitioned tables (Navis via Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a71edcf6
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a71edcf6
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a71edcf6
Branch: refs/heads/llap
Commit: a71edcf6a5672452a8e00c2bad4f20cffced26d9
Parents: 3b6b56d
Author: Navis Ryu <na...@apache.org>
Authored: Sun Feb 8 17:57:00 2015 -0800
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Thu Mar 24 19:09:47 2016 -0700
----------------------------------------------------------------------
.../ql/optimizer/stats/annotation/StatsRulesProcFactory.java | 3 +--
.../java/org/apache/hadoop/hive/ql/parse/ParseContext.java | 5 +++++
.../org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java | 8 ++++++--
3 files changed, 12 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/a71edcf6/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
index 4bcf6bf..c4fc5ca 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
@@ -105,8 +105,7 @@ public class StatsRulesProcFactory {
Object... nodeOutputs) throws SemanticException {
TableScanOperator tsop = (TableScanOperator) nd;
AnnotateStatsProcCtx aspCtx = (AnnotateStatsProcCtx) procCtx;
- PrunedPartitionList partList =
- aspCtx.getParseContext().getPrunedPartitions(tsop.getName(), tsop);
+ PrunedPartitionList partList = aspCtx.getParseContext().getPrunedPartitions(tsop);
Table table = tsop.getConf().getTableMetadata();
try {
http://git-wip-us.apache.org/repos/asf/hive/blob/a71edcf6/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java
index 4f784d1..95c254c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java
@@ -466,6 +466,11 @@ public class ParseContext {
this.fetchTask = fetchTask;
}
+ public PrunedPartitionList getPrunedPartitions(TableScanOperator ts)
+ throws SemanticException {
+ return getPrunedPartitions(ts.getConf().getAlias(), ts);
+ }
+
public PrunedPartitionList getPrunedPartitions(String alias, TableScanOperator ts)
throws SemanticException {
PrunedPartitionList partsList = opToPartList.get(ts);
http://git-wip-us.apache.org/repos/asf/hive/blob/a71edcf6/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index d9db1d5..adee14b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -10776,10 +10776,14 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
// check whether any of them break the limit
for (Operator<?> topOp : topOps.values()) {
if (topOp instanceof TableScanOperator) {
- if (((TableScanDesc)topOp.getConf()).getIsMetadataOnly()) {
+ TableScanOperator tsOp = (TableScanOperator) topOp;
+ if (tsOp.getConf().getIsMetadataOnly()) {
+ continue;
+ }
+ PrunedPartitionList parts = pCtx.getPrunedPartitions(tsOp);
+ if (!parts.getSourceTable().isPartitioned()) {
continue;
}
- PrunedPartitionList parts = pCtx.getOpToPartList().get(topOp);
if (parts.getPartitions().size() > scanLimit) {
throw new SemanticException(ErrorMsg.PARTITION_SCAN_LIMIT_EXCEEDED, ""
+ parts.getPartitions().size(), "" + parts.getSourceTable().getTableName(), ""
[41/50] [abbrv] hive git commit: HIVE-12988 : Improve dynamic
partition loading IV (Ashutosh Chauhan via Prasanth J)
Posted by jd...@apache.org.
HIVE-12988 : Improve dynamic partition loading IV (Ashutosh Chauhan via Prasanth J)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a14ef8ab
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a14ef8ab
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a14ef8ab
Branch: refs/heads/llap
Commit: a14ef8abe1df1516b8b9f486030bc3d584f940a9
Parents: 1de97bc
Author: Ashutosh Chauhan <ha...@apache.org>
Authored: Tue Feb 2 18:03:44 2016 -0800
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Tue Mar 29 11:27:12 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 4 +-
.../apache/hadoop/hive/ql/metadata/Hive.java | 252 +++++++++++--------
.../org/apache/hadoop/fs/ProxyFileSystem.java | 5 +-
3 files changed, 155 insertions(+), 106 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/a14ef8ab/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index b8870f2..f03c1ab 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2326,6 +2326,8 @@ public class HiveConf extends Configuration {
HIVE_SECURITY_COMMAND_WHITELIST("hive.security.command.whitelist", "set,reset,dfs,add,list,delete,reload,compile",
"Comma separated list of non-SQL Hive commands users are authorized to execute"),
+ HIVE_MOVE_FILES_THREAD_COUNT("hive.mv.files.thread", 25, new SizeValidator(1L, true, 1024L, true), "Number of threads"
+ + " used to move files in move task"),
// If this is set all move tasks at the end of a multi-insert query will only begin once all
// outputs are ready
HIVE_MULTI_INSERT_MOVE_TASKS_SHARE_DEPENDENCIES(
@@ -2771,7 +2773,7 @@ public class HiveConf extends Configuration {
SPARK_RPC_SASL_MECHANISM("hive.spark.client.rpc.sasl.mechanisms", "DIGEST-MD5",
"Name of the SASL mechanism to use for authentication."),
SPARK_RPC_SERVER_ADDRESS("hive.spark.client.rpc.server.address", "",
- "The server address of HiverServer2 host to be used for communication between Hive client and remote Spark driver. " +
+ "The server address of HiverServer2 host to be used for communication between Hive client and remote Spark driver. " +
"Default is empty, which means the address will be determined in the same way as for hive.server2.thrift.bind.host." +
"This is only necessary if the host has mutiple network addresses and if a different network address other than " +
"hive.server2.thrift.bind.host is to be used."),
http://git-wip-us.apache.org/repos/asf/hive/blob/a14ef8ab/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 6d27f55..c27481f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -32,19 +32,25 @@ import java.io.IOException;
import java.io.PrintStream;
import java.nio.ByteBuffer;
import java.util.ArrayList;
-import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicInteger;
import com.google.common.collect.ImmutableMap;
+
import javax.jdo.JDODataStoreException;
import org.apache.hadoop.conf.Configuration;
@@ -132,6 +138,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
/**
* This class has functions that implement meta data/DDL operations using calls
@@ -1504,7 +1511,7 @@ public class Hive {
isSrcLocal);
} else {
if (conf.getBoolVar(ConfVars.FIRE_EVENTS_FOR_DML) && !tbl.isTemporary() && oldPart != null) {
- newFiles = new ArrayList<>();
+ newFiles = Collections.synchronizedList(new ArrayList<Path>());
}
FileSystem fs = tbl.getDataLocation().getFileSystem(conf);
@@ -1751,9 +1758,13 @@ private void constructOneLBLocationMap(FileStatus fSta,
public void loadTable(Path loadPath, String tableName, boolean replace,
boolean isSrcLocal, boolean isSkewedStoreAsSubdir, boolean isAcid)
throws HiveException {
- List<Path> newFiles = new ArrayList<Path>();
+
+ List<Path> newFiles = null;
Table tbl = getTable(tableName);
HiveConf sessionConf = SessionState.getSessionConf();
+ if (conf.getBoolVar(ConfVars.FIRE_EVENTS_FOR_DML) && !tbl.isTemporary()) {
+ newFiles = Collections.synchronizedList(new ArrayList<Path>());
+ }
if (replace) {
Path tableDest = tbl.getPath();
replaceFiles(tableDest, loadPath, tableDest, tableDest, sessionConf, isSrcLocal);
@@ -2579,75 +2590,91 @@ private void constructOneLBLocationMap(FileStatus fSta,
}
}
- // for each file or directory in 'srcs', make mapping for every file in src to safe name in dest
- private static List<List<Path[]>> checkPaths(HiveConf conf, FileSystem fs,
- FileStatus[] srcs, FileSystem srcFs, Path destf, boolean replace)
- throws HiveException {
+ private static void copyFiles(final HiveConf conf, final FileSystem destFs,
+ FileStatus[] srcs, final FileSystem srcFs, final Path destf, final boolean isSrcLocal, final List<Path> newFiles)
+ throws HiveException {
- List<List<Path[]>> result = new ArrayList<List<Path[]>>();
+ final HadoopShims.HdfsFileStatus fullDestStatus;
try {
- FileStatus destStatus = !replace ? FileUtils.getFileStatusOrNull(fs, destf) : null;
- if (destStatus != null && !destStatus.isDir()) {
- throw new HiveException("checkPaths: destination " + destf
- + " should be a directory");
- }
- for (FileStatus src : srcs) {
- FileStatus[] items;
- if (src.isDir()) {
- items = srcFs.listStatus(src.getPath(), FileUtils.HIDDEN_FILES_PATH_FILTER);
- Arrays.sort(items);
+ fullDestStatus = ShimLoader.getHadoopShims().getFullFileStatus(conf, destFs, destf);
+ } catch (IOException e1) {
+ throw new HiveException(e1);
+ }
+
+ if (!fullDestStatus.getFileStatus().isDirectory()) {
+ throw new HiveException(destf + " is not a directory.");
+ }
+ final boolean inheritPerms = HiveConf.getBoolVar(conf,
+ HiveConf.ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS);
+ final List<Future<ObjectPair<Path, Path>>> futures = new LinkedList<>();
+ final ExecutorService pool = Executors.newFixedThreadPool(
+ conf.getIntVar(ConfVars.HIVE_MOVE_FILES_THREAD_COUNT),
+ new ThreadFactoryBuilder().setDaemon(true).setNameFormat("MoveDir-Thread-%d").build());
+
+ for (FileStatus src : srcs) {
+ FileStatus[] files;
+ if (src.isDirectory()) {
+ try {
+ files = srcFs.listStatus(src.getPath(), FileUtils.HIDDEN_FILES_PATH_FILTER);
+ } catch (IOException e) {
+ pool.shutdownNow();
+ throw new HiveException(e);
+ }
+ } else {
+ files = new FileStatus[] {src};
+ }
+
+ for (FileStatus srcFile : files) {
+
+ final Path srcP = srcFile.getPath();
+ final boolean needToCopy = needToCopy(srcP, destf, srcFs, destFs);
+ // Strip off the file type, if any so we don't make:
+ // 000000_0.gz -> 000000_0.gz_copy_1
+ final String name;
+ final String filetype;
+ String itemName = srcP.getName();
+ int index = itemName.lastIndexOf('.');
+ if (index >= 0) {
+ filetype = itemName.substring(index);
+ name = itemName.substring(0, index);
} else {
- items = new FileStatus[] {src};
+ name = itemName;
+ filetype = "";
}
-
- List<Path[]> srcToDest = new ArrayList<Path[]>();
- for (FileStatus item : items) {
-
- Path itemSource = item.getPath();
-
- if (Utilities.isTempPath(item)) {
- // This check is redundant because temp files are removed by
- // execution layer before
- // calling loadTable/Partition. But leaving it in just in case.
- srcFs.delete(itemSource, true);
- continue;
- }
-
- Path itemDest = new Path(destf, itemSource.getName());
-
- if (!replace) {
- // Strip off the file type, if any so we don't make:
- // 000000_0.gz -> 000000_0.gz_copy_1
- String name = itemSource.getName();
- String filetype;
- int index = name.lastIndexOf('.');
- if (index >= 0) {
- filetype = name.substring(index);
- name = name.substring(0, index);
+ futures.add(pool.submit(new Callable<ObjectPair<Path, Path>>() {
+ @Override
+ public ObjectPair<Path, Path> call() throws Exception {
+ Path destPath = new Path(destf, srcP.getName());
+ if (!needToCopy && !isSrcLocal) {
+ for (int counter = 1; !destFs.rename(srcP,destPath); counter++) {
+ destPath = new Path(destf, name + ("_copy_" + counter) + filetype);
+ }
} else {
- filetype = "";
+ destPath = mvFile(conf, srcP, destPath, isSrcLocal, srcFs, destFs, name, filetype);
}
- // It's possible that the file we're copying may have the same
- // relative name as an existing file in the "destf" directory.
- // So let's make a quick check to see if we can rename any
- // potential offenders so as to allow them to move into the
- // "destf" directory. The scheme is dead simple: simply tack
- // on "_copy_N" where N starts at 1 and works its way up until
- // we find a free space.
-
- // removed source file staging.. it's more confusing when failed.
- for (int counter = 1; fs.exists(itemDest) || destExists(result, itemDest); counter++) {
- itemDest = new Path(destf, name + ("_copy_" + counter) + filetype);
+
+ if (inheritPerms) {
+ ShimLoader.getHadoopShims().setFullFileStatus(conf, fullDestStatus, destFs, destf);
}
+ if (null != newFiles) {
+ newFiles.add(destPath);
+ }
+ return ObjectPair.create(srcP, destPath);
}
- srcToDest.add(new Path[]{itemSource, itemDest});
- }
- result.add(srcToDest);
+ }));
+ }
+ }
+ pool.shutdown();
+ for (Future<ObjectPair<Path, Path>> future : futures) {
+ try {
+ ObjectPair<Path, Path> pair = future.get();
+ LOG.debug("Moved src: {}", pair.getFirst().toString(), ", to dest: {}", pair.getSecond().toString());
+ } catch (Exception e) {
+ LOG.error("Failed to move: {}", e.getMessage());
+ pool.shutdownNow();
+ throw new HiveException(e.getCause());
}
- } catch (IOException e) {
- throw new HiveException("checkPaths: filesystem error in check phase. " + e.getMessage(), e);
}
- return result;
}
private static boolean destExists(List<List<Path[]>> result, Path proposed) {
@@ -2704,14 +2731,34 @@ private void constructOneLBLocationMap(FileStatus fSta,
return ShimLoader.getHadoopShims().getPathWithoutSchemeAndAuthority(path).toString();
}
+ private static Path mvFile(HiveConf conf, Path srcf, Path destf, boolean isSrcLocal,
+ FileSystem srcFs, FileSystem destFs, String srcName, String filetype) throws IOException {
+
+ for (int counter = 1; destFs.exists(destf); counter++) {
+ destf = new Path(destf.getParent(), srcName + ("_copy_" + counter) + filetype);
+ }
+ if (isSrcLocal) {
+ // For local src file, copy to hdfs
+ destFs.copyFromLocalFile(srcf, destf);
+ } else {
+ //copy if across file system or encryption zones.
+ LOG.info("Copying source " + srcf + " to " + destf + " because HDFS encryption zones are different.");
+ FileUtils.copy(srcFs, srcf, destFs, destf,
+ true, // delete source
+ false, // overwrite destination
+ conf);
+ }
+ return destf;
+ }
+
//it is assumed that parent directory of the destf should already exist when this
//method is called. when the replace value is true, this method works a little different
//from mv command if the destf is a directory, it replaces the destf instead of moving under
//the destf. in this case, the replaced destf still preserves the original destf's permission
- public static boolean moveFile(HiveConf conf, Path srcf, Path destf,
+ public static boolean moveFile(HiveConf conf, Path srcf, final Path destf,
boolean replace, boolean isSrcLocal) throws HiveException {
boolean success = false;
- FileSystem srcFs, destFs;
+ final FileSystem srcFs, destFs;
try {
destFs = destf.getFileSystem(conf);
} catch (IOException e) {
@@ -2775,31 +2822,38 @@ private void constructOneLBLocationMap(FileStatus fSta,
FileStatus[] srcs = destFs.listStatus(srcf, FileUtils.HIDDEN_FILES_PATH_FILTER);
if (srcs.length == 0) {
success = true; // Nothing to move.
- }
- /* Move files one by one because source is a subdirectory of destination */
- for (FileStatus status : srcs) {
- Path destFile;
-
- /* Append the source filename to the destination directory */
- if (destFs.isDirectory(destf)) {
- destFile = new Path(destf, status.getPath().getName());
- } else {
- destFile = destf;
+ } else {
+ List<Future<Boolean>> futures = new LinkedList<>();
+ final ExecutorService pool = Executors.newFixedThreadPool(
+ conf.getIntVar(ConfVars.HIVE_MOVE_FILES_THREAD_COUNT),
+ new ThreadFactoryBuilder().setDaemon(true).setNameFormat("MoveDir-Thread-%d").build());
+ /* Move files one by one because source is a subdirectory of destination */
+ for (final FileStatus status : srcs) {
+ futures.add(pool.submit(new Callable<Boolean>() {
+ @Override
+ public Boolean call() throws Exception {
+ return destFs.rename(status.getPath(), destf);
+ }
+ }));
}
-
- // Destination should be replaced, so we delete it first
- if (destFs.exists(destFile)) {
- if (!destFs.delete(destFile, true)) {
- throw new HiveException(String.format("File to replace could not be deleted: %s", destFile));
+ pool.shutdown();
+ boolean allFutures = true;
+ for (Future<Boolean> future : futures) {
+ try {
+ Boolean result = future.get();
+ allFutures &= result;
+ if (!result) {
+ LOG.debug("Failed to rename.");
+ pool.shutdownNow();
+ }
+ } catch (Exception e) {
+ LOG.debug("Failed to rename.", e.getMessage());
+ pool.shutdownNow();
+ throw new HiveException(e.getCause());
}
}
-
- if (!(destFs.rename(status.getPath(), destFile))) {
- throw new HiveException("Unable to move source " + status.getPath() + " to destination " + destf);
- }
+ success = allFutures;
}
-
- success = true;
} else {
success = destFs.rename(srcf, destf);
}
@@ -2825,8 +2879,9 @@ private void constructOneLBLocationMap(FileStatus fSta,
/**
* If moving across different FileSystems or differnent encryption zone, need to do a File copy instead of rename.
* TODO- consider if need to do this for different file authority.
+ * @throws HiveException
*/
- static protected boolean needToCopy(Path srcf, Path destf, FileSystem srcFs, FileSystem destFs) throws HiveException, IOException {
+ static protected boolean needToCopy(Path srcf, Path destf, FileSystem srcFs, FileSystem destFs) throws HiveException {
//Check if different FileSystems
if (!srcFs.getClass().equals(destFs.getClass())) {
return true;
@@ -2834,8 +2889,12 @@ private void constructOneLBLocationMap(FileStatus fSta,
//Check if different encryption zones
HadoopShims.HdfsEncryptionShim hdfsEncryptionShim = SessionState.get().getHdfsEncryptionShim();
- return hdfsEncryptionShim != null && (hdfsEncryptionShim.isPathEncrypted(srcf) || hdfsEncryptionShim.isPathEncrypted(destf))
- && !hdfsEncryptionShim.arePathsOnSameEncryptionZone(srcf, destf);
+ try {
+ return hdfsEncryptionShim != null && (hdfsEncryptionShim.isPathEncrypted(srcf) || hdfsEncryptionShim.isPathEncrypted(destf))
+ && !hdfsEncryptionShim.arePathsOnSameEncryptionZone(srcf, destf);
+ } catch (IOException e) {
+ throw new HiveException(e);
+ }
}
/**
@@ -2886,22 +2945,7 @@ private void constructOneLBLocationMap(FileStatus fSta,
if (isAcid) {
moveAcidFiles(srcFs, srcs, destf, newFiles);
} else {
- // check that source and target paths exist
- List<List<Path[]>> result = checkPaths(conf, fs, srcs, srcFs, destf, false);
- // move it, move it
- try {
- for (List<Path[]> sdpairs : result) {
- for (Path[] sdpair : sdpairs) {
- if (!moveFile(conf, sdpair[0], sdpair[1], false, isSrcLocal)) {
- throw new IOException("Cannot move " + sdpair[0] + " to "
- + sdpair[1]);
- }
- if (newFiles != null) newFiles.add(sdpair[1]);
- }
- }
- } catch (IOException e) {
- throw new HiveException("copyFiles: error while moving files!!! " + e.getMessage(), e);
- }
+ copyFiles(conf, fs, srcs, srcFs, destf, isSrcLocal, newFiles);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a14ef8ab/shims/common/src/main/java/org/apache/hadoop/fs/ProxyFileSystem.java
----------------------------------------------------------------------
diff --git a/shims/common/src/main/java/org/apache/hadoop/fs/ProxyFileSystem.java b/shims/common/src/main/java/org/apache/hadoop/fs/ProxyFileSystem.java
index cb1e2b7..2c37a51 100644
--- a/shims/common/src/main/java/org/apache/hadoop/fs/ProxyFileSystem.java
+++ b/shims/common/src/main/java/org/apache/hadoop/fs/ProxyFileSystem.java
@@ -82,6 +82,7 @@ public class ProxyFileSystem extends FilterFileSystem {
* @return
* @throws IOException
*/
+ @Override
public Path resolvePath(final Path p) throws IOException {
// Return the fully-qualified path of path f resolving the path
// through any symlinks or mount point
@@ -174,7 +175,9 @@ public class ProxyFileSystem extends FilterFileSystem {
@Override
public boolean rename(Path src, Path dst) throws IOException {
- return super.rename(swizzleParamPath(src), swizzleParamPath(dst));
+ Path dest = swizzleParamPath(dst);
+ // Make sure for existing destination we return false as per FileSystem api contract
+ return super.isFile(dest) ? false : super.rename(swizzleParamPath(src), dest);
}
@Override
[25/50] [abbrv] hive git commit: HIVE-13111: Fix timestamp /
interval_day_time wrong results with HIVE-9862 (Matt McCline,
reviewed by Jason Dere)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/test/results/clientpositive/vector_interval_arithmetic.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_interval_arithmetic.q.out b/ql/src/test/results/clientpositive/vector_interval_arithmetic.q.out
new file mode 100644
index 0000000..cd8111d
--- /dev/null
+++ b/ql/src/test/results/clientpositive/vector_interval_arithmetic.q.out
@@ -0,0 +1,1027 @@
+PREHOOK: query: create table unique_timestamps (tsval timestamp) STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@unique_timestamps
+POSTHOOK: query: create table unique_timestamps (tsval timestamp) STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@unique_timestamps
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/timestamps.txt' OVERWRITE INTO TABLE unique_timestamps
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@unique_timestamps
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/timestamps.txt' OVERWRITE INTO TABLE unique_timestamps
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@unique_timestamps
+PREHOOK: query: create table interval_arithmetic_1 (dateval date, tsval timestamp) stored as orc
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@interval_arithmetic_1
+POSTHOOK: query: create table interval_arithmetic_1 (dateval date, tsval timestamp) stored as orc
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@interval_arithmetic_1
+PREHOOK: query: insert overwrite table interval_arithmetic_1
+ select cast(tsval as date), tsval from unique_timestamps
+PREHOOK: type: QUERY
+PREHOOK: Input: default@unique_timestamps
+PREHOOK: Output: default@interval_arithmetic_1
+POSTHOOK: query: insert overwrite table interval_arithmetic_1
+ select cast(tsval as date), tsval from unique_timestamps
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@unique_timestamps
+POSTHOOK: Output: default@interval_arithmetic_1
+POSTHOOK: Lineage: interval_arithmetic_1.dateval EXPRESSION [(unique_timestamps)unique_timestamps.FieldSchema(name:tsval, type:timestamp, comment:null), ]
+POSTHOOK: Lineage: interval_arithmetic_1.tsval SIMPLE [(unique_timestamps)unique_timestamps.FieldSchema(name:tsval, type:timestamp, comment:null), ]
+_c0 tsval
+PREHOOK: query: -- interval year-month arithmetic
+explain
+select
+ dateval,
+ dateval - interval '2-2' year to month,
+ dateval - interval '-2-2' year to month,
+ dateval + interval '2-2' year to month,
+ dateval + interval '-2-2' year to month,
+ - interval '2-2' year to month + dateval,
+ interval '2-2' year to month + dateval
+from interval_arithmetic_1
+order by dateval
+PREHOOK: type: QUERY
+POSTHOOK: query: -- interval year-month arithmetic
+explain
+select
+ dateval,
+ dateval - interval '2-2' year to month,
+ dateval - interval '-2-2' year to month,
+ dateval + interval '2-2' year to month,
+ dateval + interval '-2-2' year to month,
+ - interval '2-2' year to month + dateval,
+ interval '2-2' year to month + dateval
+from interval_arithmetic_1
+order by dateval
+POSTHOOK: type: QUERY
+Explain
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: interval_arithmetic_1
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: dateval (type: date), (dateval - 2-2) (type: date), (dateval - -2-2) (type: date), (dateval + 2-2) (type: date), (dateval + -2-2) (type: date), (-2-2 + dateval) (type: date), (2-2 + dateval) (type: date)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: date)
+ sort order: +
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: date), _col4 (type: date), _col5 (type: date), _col6 (type: date)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: date), VALUE._col0 (type: date), VALUE._col1 (type: date), VALUE._col2 (type: date), VALUE._col3 (type: date), VALUE._col4 (type: date), VALUE._col5 (type: date)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select
+ dateval,
+ dateval - interval '2-2' year to month,
+ dateval - interval '-2-2' year to month,
+ dateval + interval '2-2' year to month,
+ dateval + interval '-2-2' year to month,
+ - interval '2-2' year to month + dateval,
+ interval '2-2' year to month + dateval
+from interval_arithmetic_1
+order by dateval
+PREHOOK: type: QUERY
+PREHOOK: Input: default@interval_arithmetic_1
+#### A masked pattern was here ####
+POSTHOOK: query: select
+ dateval,
+ dateval - interval '2-2' year to month,
+ dateval - interval '-2-2' year to month,
+ dateval + interval '2-2' year to month,
+ dateval + interval '-2-2' year to month,
+ - interval '2-2' year to month + dateval,
+ interval '2-2' year to month + dateval
+from interval_arithmetic_1
+order by dateval
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@interval_arithmetic_1
+#### A masked pattern was here ####
+dateval c1 c2 c3 c4 c5 c6
+0004-09-22 0002-07-22 0006-11-22 0006-11-22 0002-07-22 0002-07-22 0006-11-22
+0528-10-27 0526-08-27 0530-12-27 0530-12-27 0526-08-27 0526-08-27 0530-12-27
+1319-02-02 1316-12-02 1321-04-02 1321-04-02 1316-12-02 1316-12-02 1321-04-02
+1404-07-23 1402-05-23 1406-09-23 1406-09-23 1402-05-23 1402-05-23 1406-09-23
+1815-05-06 1813-03-06 1817-07-06 1817-07-06 1813-03-06 1813-03-06 1817-07-06
+1883-04-17 1881-02-17 1885-06-17 1885-06-17 1881-02-17 1881-02-17 1885-06-17
+1966-08-16 1964-06-16 1968-10-16 1968-10-16 1964-06-16 1964-06-16 1968-10-16
+1973-04-17 1971-02-17 1975-06-17 1975-06-17 1971-02-17 1971-02-17 1975-06-17
+1974-10-04 1972-08-04 1976-12-04 1976-12-04 1972-08-04 1972-08-04 1976-12-04
+1976-03-03 1974-01-03 1978-05-03 1978-05-03 1974-01-03 1974-01-03 1978-05-03
+1976-05-06 1974-03-06 1978-07-06 1978-07-06 1974-03-06 1974-03-06 1978-07-06
+1978-08-05 1976-06-05 1980-10-05 1980-10-05 1976-06-05 1976-06-05 1980-10-05
+1981-04-25 1979-02-25 1983-06-25 1983-06-25 1979-02-25 1979-02-25 1983-06-25
+1981-11-15 1979-09-15 1984-01-15 1984-01-15 1979-09-15 1979-09-15 1984-01-15
+1985-07-20 1983-05-20 1987-09-20 1987-09-20 1983-05-20 1983-05-20 1987-09-20
+1985-11-18 1983-09-18 1988-01-18 1988-01-18 1983-09-18 1983-09-18 1988-01-18
+1987-02-21 1984-12-21 1989-04-21 1989-04-21 1984-12-21 1984-12-21 1989-04-21
+1987-05-28 1985-03-28 1989-07-28 1989-07-28 1985-03-28 1985-03-28 1989-07-28
+1998-10-16 1996-08-16 2000-12-16 2000-12-16 1996-08-16 1996-08-16 2000-12-16
+1999-10-03 1997-08-03 2001-12-03 2001-12-03 1997-08-03 1997-08-03 2001-12-03
+2000-12-18 1998-10-18 2003-02-18 2003-02-18 1998-10-18 1998-10-18 2003-02-18
+2002-05-10 2000-03-10 2004-07-10 2004-07-10 2000-03-10 2000-03-10 2004-07-10
+2003-09-23 2001-07-23 2005-11-23 2005-11-23 2001-07-23 2001-07-23 2005-11-23
+2004-03-07 2002-01-07 2006-05-07 2006-05-07 2002-01-07 2002-01-07 2006-05-07
+2007-02-09 2004-12-09 2009-04-09 2009-04-09 2004-12-09 2004-12-09 2009-04-09
+2009-01-21 2006-11-21 2011-03-21 2011-03-21 2006-11-21 2006-11-21 2011-03-21
+2010-04-08 2008-02-08 2012-06-08 2012-06-08 2008-02-08 2008-02-08 2012-06-08
+2013-04-07 2011-02-07 2015-06-07 2015-06-07 2011-02-07 2011-02-07 2015-06-07
+2013-04-10 2011-02-10 2015-06-10 2015-06-10 2011-02-10 2011-02-10 2015-06-10
+2021-09-24 2019-07-24 2023-11-24 2023-11-24 2019-07-24 2019-07-24 2023-11-24
+2024-11-11 2022-09-11 2027-01-11 2027-01-11 2022-09-11 2022-09-11 2027-01-11
+4143-07-08 4141-05-08 4145-09-08 4145-09-08 4141-05-08 4141-05-08 4145-09-08
+4966-12-04 4964-10-04 4969-02-04 4969-02-04 4964-10-04 4964-10-04 4969-02-04
+5339-02-01 5336-12-01 5341-04-01 5341-04-01 5336-12-01 5336-12-01 5341-04-01
+5344-10-04 5342-08-04 5346-12-04 5346-12-04 5342-08-04 5342-08-04 5346-12-04
+5397-07-13 5395-05-13 5399-09-13 5399-09-13 5395-05-13 5395-05-13 5399-09-13
+5966-07-09 5964-05-09 5968-09-09 5968-09-09 5964-05-09 5964-05-09 5968-09-09
+6229-06-28 6227-04-28 6231-08-28 6231-08-28 6227-04-28 6227-04-28 6231-08-28
+6482-04-27 6480-02-27 6484-06-27 6484-06-27 6480-02-27 6480-02-27 6484-06-27
+6631-11-13 6629-09-13 6634-01-13 6634-01-13 6629-09-13 6629-09-13 6634-01-13
+6705-09-28 6703-07-28 6707-11-28 6707-11-28 6703-07-28 6703-07-28 6707-11-28
+6731-02-12 6728-12-12 6733-04-12 6733-04-12 6728-12-12 6728-12-12 6733-04-12
+7160-12-02 7158-10-02 7163-02-02 7163-02-02 7158-10-02 7158-10-02 7163-02-02
+7409-09-07 7407-07-07 7411-11-07 7411-11-07 7407-07-07 7407-07-07 7411-11-07
+7503-06-23 7501-04-23 7505-08-23 7505-08-23 7501-04-23 7501-04-23 7505-08-23
+8422-07-22 8420-05-22 8424-09-22 8424-09-22 8420-05-22 8420-05-22 8424-09-22
+8521-01-16 8518-11-16 8523-03-16 8523-03-16 8518-11-16 8518-11-16 8523-03-16
+9075-06-13 9073-04-13 9077-08-13 9077-08-13 9073-04-13 9073-04-13 9077-08-13
+9209-11-11 9207-09-11 9212-01-11 9212-01-11 9207-09-11 9207-09-11 9212-01-11
+9403-01-09 9400-11-09 9405-03-09 9405-03-09 9400-11-09 9400-11-09 9405-03-09
+PREHOOK: query: explain
+select
+ dateval,
+ dateval - date '1999-06-07',
+ date '1999-06-07' - dateval,
+ dateval - dateval
+from interval_arithmetic_1
+order by dateval
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select
+ dateval,
+ dateval - date '1999-06-07',
+ date '1999-06-07' - dateval,
+ dateval - dateval
+from interval_arithmetic_1
+order by dateval
+POSTHOOK: type: QUERY
+Explain
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: interval_arithmetic_1
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: dateval (type: date), (dateval - 1999-06-07) (type: interval_day_time), (1999-06-07 - dateval) (type: interval_day_time), (dateval - dateval) (type: interval_day_time)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: date)
+ sort order: +
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: interval_day_time), _col2 (type: interval_day_time), _col3 (type: interval_day_time)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: date), VALUE._col0 (type: interval_day_time), VALUE._col1 (type: interval_day_time), VALUE._col2 (type: interval_day_time)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select
+ dateval,
+ dateval - date '1999-06-07',
+ date '1999-06-07' - dateval,
+ dateval - dateval
+from interval_arithmetic_1
+order by dateval
+PREHOOK: type: QUERY
+PREHOOK: Input: default@interval_arithmetic_1
+#### A masked pattern was here ####
+POSTHOOK: query: select
+ dateval,
+ dateval - date '1999-06-07',
+ date '1999-06-07' - dateval,
+ dateval - dateval
+from interval_arithmetic_1
+order by dateval
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@interval_arithmetic_1
+#### A masked pattern was here ####
+dateval c1 c2 c3
+0004-09-22 -728552 23:00:00.000000000 728552 23:00:00.000000000 0 00:00:00.000000000
+0528-10-27 -537126 23:00:00.000000000 537126 23:00:00.000000000 0 00:00:00.000000000
+1319-02-02 -248481 23:00:00.000000000 248481 23:00:00.000000000 0 00:00:00.000000000
+1404-07-23 -217263 23:00:00.000000000 217263 23:00:00.000000000 0 00:00:00.000000000
+1815-05-06 -67236 23:00:00.000000000 67236 23:00:00.000000000 0 00:00:00.000000000
+1883-04-17 -42418 23:00:00.000000000 42418 23:00:00.000000000 0 00:00:00.000000000
+1966-08-16 -11983 00:00:00.000000000 11983 00:00:00.000000000 0 00:00:00.000000000
+1973-04-17 -9546 23:00:00.000000000 9546 23:00:00.000000000 0 00:00:00.000000000
+1974-10-04 -9012 00:00:00.000000000 9012 00:00:00.000000000 0 00:00:00.000000000
+1976-03-03 -8495 23:00:00.000000000 8495 23:00:00.000000000 0 00:00:00.000000000
+1976-05-06 -8432 00:00:00.000000000 8432 00:00:00.000000000 0 00:00:00.000000000
+1978-08-05 -7611 00:00:00.000000000 7611 00:00:00.000000000 0 00:00:00.000000000
+1981-04-25 -6616 23:00:00.000000000 6616 23:00:00.000000000 0 00:00:00.000000000
+1981-11-15 -6412 23:00:00.000000000 6412 23:00:00.000000000 0 00:00:00.000000000
+1985-07-20 -5070 00:00:00.000000000 5070 00:00:00.000000000 0 00:00:00.000000000
+1985-11-18 -4948 23:00:00.000000000 4948 23:00:00.000000000 0 00:00:00.000000000
+1987-02-21 -4488 23:00:00.000000000 4488 23:00:00.000000000 0 00:00:00.000000000
+1987-05-28 -4393 00:00:00.000000000 4393 00:00:00.000000000 0 00:00:00.000000000
+1998-10-16 -234 00:00:00.000000000 234 00:00:00.000000000 0 00:00:00.000000000
+1999-10-03 118 00:00:00.000000000 -118 00:00:00.000000000 0 00:00:00.000000000
+2000-12-18 560 01:00:00.000000000 -560 01:00:00.000000000 0 00:00:00.000000000
+2002-05-10 1068 00:00:00.000000000 -1068 00:00:00.000000000 0 00:00:00.000000000
+2003-09-23 1569 00:00:00.000000000 -1569 00:00:00.000000000 0 00:00:00.000000000
+2004-03-07 1735 01:00:00.000000000 -1735 01:00:00.000000000 0 00:00:00.000000000
+2007-02-09 2804 01:00:00.000000000 -2804 01:00:00.000000000 0 00:00:00.000000000
+2009-01-21 3516 01:00:00.000000000 -3516 01:00:00.000000000 0 00:00:00.000000000
+2010-04-08 3958 00:00:00.000000000 -3958 00:00:00.000000000 0 00:00:00.000000000
+2013-04-07 5053 00:00:00.000000000 -5053 00:00:00.000000000 0 00:00:00.000000000
+2013-04-10 5056 00:00:00.000000000 -5056 00:00:00.000000000 0 00:00:00.000000000
+2021-09-24 8145 00:00:00.000000000 -8145 00:00:00.000000000 0 00:00:00.000000000
+2024-11-11 9289 01:00:00.000000000 -9289 01:00:00.000000000 0 00:00:00.000000000
+4143-07-08 783111 00:00:00.000000000 -783111 00:00:00.000000000 0 00:00:00.000000000
+4966-12-04 1083855 01:00:00.000000000 -1083855 01:00:00.000000000 0 00:00:00.000000000
+5339-02-01 1219784 01:00:00.000000000 -1219784 01:00:00.000000000 0 00:00:00.000000000
+5344-10-04 1221856 00:00:00.000000000 -1221856 00:00:00.000000000 0 00:00:00.000000000
+5397-07-13 1241131 00:00:00.000000000 -1241131 00:00:00.000000000 0 00:00:00.000000000
+5966-07-09 1448949 00:00:00.000000000 -1448949 00:00:00.000000000 0 00:00:00.000000000
+6229-06-28 1544997 00:00:00.000000000 -1544997 00:00:00.000000000 0 00:00:00.000000000
+6482-04-27 1637342 00:00:00.000000000 -1637342 00:00:00.000000000 0 00:00:00.000000000
+6631-11-13 1691962 01:00:00.000000000 -1691962 01:00:00.000000000 0 00:00:00.000000000
+6705-09-28 1718944 00:00:00.000000000 -1718944 00:00:00.000000000 0 00:00:00.000000000
+6731-02-12 1728212 01:00:00.000000000 -1728212 01:00:00.000000000 0 00:00:00.000000000
+7160-12-02 1885195 01:00:00.000000000 -1885195 01:00:00.000000000 0 00:00:00.000000000
+7409-09-07 1976054 00:00:00.000000000 -1976054 00:00:00.000000000 0 00:00:00.000000000
+7503-06-23 2010310 00:00:00.000000000 -2010310 00:00:00.000000000 0 00:00:00.000000000
+8422-07-22 2345998 00:00:00.000000000 -2345998 00:00:00.000000000 0 00:00:00.000000000
+8521-01-16 2381970 01:00:00.000000000 -2381970 01:00:00.000000000 0 00:00:00.000000000
+9075-06-13 2584462 00:00:00.000000000 -2584462 00:00:00.000000000 0 00:00:00.000000000
+9209-11-11 2633556 01:00:00.000000000 -2633556 01:00:00.000000000 0 00:00:00.000000000
+9403-01-09 2704106 01:00:00.000000000 -2704106 01:00:00.000000000 0 00:00:00.000000000
+PREHOOK: query: explain
+select
+ tsval,
+ tsval - interval '2-2' year to month,
+ tsval - interval '-2-2' year to month,
+ tsval + interval '2-2' year to month,
+ tsval + interval '-2-2' year to month,
+ - interval '2-2' year to month + tsval,
+ interval '2-2' year to month + tsval
+from interval_arithmetic_1
+order by tsval
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select
+ tsval,
+ tsval - interval '2-2' year to month,
+ tsval - interval '-2-2' year to month,
+ tsval + interval '2-2' year to month,
+ tsval + interval '-2-2' year to month,
+ - interval '2-2' year to month + tsval,
+ interval '2-2' year to month + tsval
+from interval_arithmetic_1
+order by tsval
+POSTHOOK: type: QUERY
+Explain
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: interval_arithmetic_1
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: tsval (type: timestamp), (tsval - 2-2) (type: timestamp), (tsval - -2-2) (type: timestamp), (tsval + 2-2) (type: timestamp), (tsval + -2-2) (type: timestamp), (-2-2 + tsval) (type: timestamp), (2-2 + tsval) (type: timestamp)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: timestamp)
+ sort order: +
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: timestamp), _col2 (type: timestamp), _col3 (type: timestamp), _col4 (type: timestamp), _col5 (type: timestamp), _col6 (type: timestamp)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: timestamp), VALUE._col0 (type: timestamp), VALUE._col1 (type: timestamp), VALUE._col2 (type: timestamp), VALUE._col3 (type: timestamp), VALUE._col4 (type: timestamp), VALUE._col5 (type: timestamp)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select
+ tsval,
+ tsval - interval '2-2' year to month,
+ tsval - interval '-2-2' year to month,
+ tsval + interval '2-2' year to month,
+ tsval + interval '-2-2' year to month,
+ - interval '2-2' year to month + tsval,
+ interval '2-2' year to month + tsval
+from interval_arithmetic_1
+order by tsval
+PREHOOK: type: QUERY
+PREHOOK: Input: default@interval_arithmetic_1
+#### A masked pattern was here ####
+POSTHOOK: query: select
+ tsval,
+ tsval - interval '2-2' year to month,
+ tsval - interval '-2-2' year to month,
+ tsval + interval '2-2' year to month,
+ tsval + interval '-2-2' year to month,
+ - interval '2-2' year to month + tsval,
+ interval '2-2' year to month + tsval
+from interval_arithmetic_1
+order by tsval
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@interval_arithmetic_1
+#### A masked pattern was here ####
+tsval c1 c2 c3 c4 c5 c6
+0004-09-22 18:26:29.519542222 0002-07-22 18:26:29.519542222 0006-11-22 18:26:29.519542222 0006-11-22 18:26:29.519542222 0002-07-22 18:26:29.519542222 0002-07-22 18:26:29.519542222 0006-11-22 18:26:29.519542222
+0528-10-27 08:15:18.941718273 0526-08-27 08:15:18.941718273 0530-12-27 08:15:18.941718273 0530-12-27 08:15:18.941718273 0526-08-27 08:15:18.941718273 0526-08-27 08:15:18.941718273 0530-12-27 08:15:18.941718273
+1319-02-02 16:31:57.778 1316-12-02 16:31:57.778 1321-04-02 16:31:57.778 1321-04-02 16:31:57.778 1316-12-02 16:31:57.778 1316-12-02 16:31:57.778 1321-04-02 16:31:57.778
+1404-07-23 15:32:16.059185026 1402-05-23 15:32:16.059185026 1406-09-23 15:32:16.059185026 1406-09-23 15:32:16.059185026 1402-05-23 15:32:16.059185026 1402-05-23 15:32:16.059185026 1406-09-23 15:32:16.059185026
+1815-05-06 00:12:37.543584705 1813-03-06 00:12:37.543584705 1817-07-06 00:12:37.543584705 1817-07-06 00:12:37.543584705 1813-03-06 00:12:37.543584705 1813-03-06 00:12:37.543584705 1817-07-06 00:12:37.543584705
+1883-04-17 04:14:34.647766229 1881-02-17 04:14:34.647766229 1885-06-17 04:14:34.647766229 1885-06-17 04:14:34.647766229 1881-02-17 04:14:34.647766229 1881-02-17 04:14:34.647766229 1885-06-17 04:14:34.647766229
+1966-08-16 13:36:50.183618031 1964-06-16 13:36:50.183618031 1968-10-16 13:36:50.183618031 1968-10-16 13:36:50.183618031 1964-06-16 13:36:50.183618031 1964-06-16 13:36:50.183618031 1968-10-16 13:36:50.183618031
+1973-04-17 06:30:38.596784156 1971-02-17 06:30:38.596784156 1975-06-17 07:30:38.596784156 1975-06-17 07:30:38.596784156 1971-02-17 06:30:38.596784156 1971-02-17 06:30:38.596784156 1975-06-17 07:30:38.596784156
+1974-10-04 17:21:03.989 1972-08-04 17:21:03.989 1976-12-04 16:21:03.989 1976-12-04 16:21:03.989 1972-08-04 17:21:03.989 1972-08-04 17:21:03.989 1976-12-04 16:21:03.989
+1976-03-03 04:54:33.000895162 1974-01-03 04:54:33.000895162 1978-05-03 05:54:33.000895162 1978-05-03 05:54:33.000895162 1974-01-03 04:54:33.000895162 1974-01-03 04:54:33.000895162 1978-05-03 05:54:33.000895162
+1976-05-06 00:42:30.910786948 1974-03-06 00:42:30.910786948 1978-07-06 00:42:30.910786948 1978-07-06 00:42:30.910786948 1974-03-06 00:42:30.910786948 1974-03-06 00:42:30.910786948 1978-07-06 00:42:30.910786948
+1978-08-05 14:41:05.501 1976-06-05 14:41:05.501 1980-10-05 14:41:05.501 1980-10-05 14:41:05.501 1976-06-05 14:41:05.501 1976-06-05 14:41:05.501 1980-10-05 14:41:05.501
+1981-04-25 09:01:12.077192689 1979-02-25 09:01:12.077192689 1983-06-25 10:01:12.077192689 1983-06-25 10:01:12.077192689 1979-02-25 09:01:12.077192689 1979-02-25 09:01:12.077192689 1983-06-25 10:01:12.077192689
+1981-11-15 23:03:10.999338387 1979-09-16 00:03:10.999338387 1984-01-15 23:03:10.999338387 1984-01-15 23:03:10.999338387 1979-09-16 00:03:10.999338387 1979-09-16 00:03:10.999338387 1984-01-15 23:03:10.999338387
+1985-07-20 09:30:11 1983-05-20 09:30:11 1987-09-20 09:30:11 1987-09-20 09:30:11 1983-05-20 09:30:11 1983-05-20 09:30:11 1987-09-20 09:30:11
+1985-11-18 16:37:54 1983-09-18 17:37:54 1988-01-18 16:37:54 1988-01-18 16:37:54 1983-09-18 17:37:54 1983-09-18 17:37:54 1988-01-18 16:37:54
+1987-02-21 19:48:29 1984-12-21 19:48:29 1989-04-21 20:48:29 1989-04-21 20:48:29 1984-12-21 19:48:29 1984-12-21 19:48:29 1989-04-21 20:48:29
+1987-05-28 13:52:07.900916635 1985-03-28 12:52:07.900916635 1989-07-28 13:52:07.900916635 1989-07-28 13:52:07.900916635 1985-03-28 12:52:07.900916635 1985-03-28 12:52:07.900916635 1989-07-28 13:52:07.900916635
+1998-10-16 20:05:29.397591987 1996-08-16 20:05:29.397591987 2000-12-16 19:05:29.397591987 2000-12-16 19:05:29.397591987 1996-08-16 20:05:29.397591987 1996-08-16 20:05:29.397591987 2000-12-16 19:05:29.397591987
+1999-10-03 16:59:10.396903939 1997-08-03 16:59:10.396903939 2001-12-03 15:59:10.396903939 2001-12-03 15:59:10.396903939 1997-08-03 16:59:10.396903939 1997-08-03 16:59:10.396903939 2001-12-03 15:59:10.396903939
+2000-12-18 08:42:30.000595596 1998-10-18 09:42:30.000595596 2003-02-18 08:42:30.000595596 2003-02-18 08:42:30.000595596 1998-10-18 09:42:30.000595596 1998-10-18 09:42:30.000595596 2003-02-18 08:42:30.000595596
+2002-05-10 05:29:48.990818073 2000-03-10 04:29:48.990818073 2004-07-10 05:29:48.990818073 2004-07-10 05:29:48.990818073 2000-03-10 04:29:48.990818073 2000-03-10 04:29:48.990818073 2004-07-10 05:29:48.990818073
+2003-09-23 22:33:17.00003252 2001-07-23 22:33:17.00003252 2005-11-23 21:33:17.00003252 2005-11-23 21:33:17.00003252 2001-07-23 22:33:17.00003252 2001-07-23 22:33:17.00003252 2005-11-23 21:33:17.00003252
+2004-03-07 20:14:13 2002-01-07 20:14:13 2006-05-07 21:14:13 2006-05-07 21:14:13 2002-01-07 20:14:13 2002-01-07 20:14:13 2006-05-07 21:14:13
+2007-02-09 05:17:29.368756876 2004-12-09 05:17:29.368756876 2009-04-09 06:17:29.368756876 2009-04-09 06:17:29.368756876 2004-12-09 05:17:29.368756876 2004-12-09 05:17:29.368756876 2009-04-09 06:17:29.368756876
+2009-01-21 10:49:07.108 2006-11-21 10:49:07.108 2011-03-21 11:49:07.108 2011-03-21 11:49:07.108 2006-11-21 10:49:07.108 2006-11-21 10:49:07.108 2011-03-21 11:49:07.108
+2010-04-08 02:43:35.861742727 2008-02-08 01:43:35.861742727 2012-06-08 02:43:35.861742727 2012-06-08 02:43:35.861742727 2008-02-08 01:43:35.861742727 2008-02-08 01:43:35.861742727 2012-06-08 02:43:35.861742727
+2013-04-07 02:44:43.00086821 2011-02-07 01:44:43.00086821 2015-06-07 02:44:43.00086821 2015-06-07 02:44:43.00086821 2011-02-07 01:44:43.00086821 2011-02-07 01:44:43.00086821 2015-06-07 02:44:43.00086821
+2013-04-10 00:43:46.854731546 2011-02-09 23:43:46.854731546 2015-06-10 00:43:46.854731546 2015-06-10 00:43:46.854731546 2011-02-09 23:43:46.854731546 2011-02-09 23:43:46.854731546 2015-06-10 00:43:46.854731546
+2021-09-24 03:18:32.413655165 2019-07-24 03:18:32.413655165 2023-11-24 02:18:32.413655165 2023-11-24 02:18:32.413655165 2019-07-24 03:18:32.413655165 2019-07-24 03:18:32.413655165 2023-11-24 02:18:32.413655165
+2024-11-11 16:42:41.101 2022-09-11 17:42:41.101 2027-01-11 16:42:41.101 2027-01-11 16:42:41.101 2022-09-11 17:42:41.101 2022-09-11 17:42:41.101 2027-01-11 16:42:41.101
+4143-07-08 10:53:27.252802259 4141-05-08 10:53:27.252802259 4145-09-08 10:53:27.252802259 4145-09-08 10:53:27.252802259 4141-05-08 10:53:27.252802259 4141-05-08 10:53:27.252802259 4145-09-08 10:53:27.252802259
+4966-12-04 09:30:55.202 4964-10-04 10:30:55.202 4969-02-04 09:30:55.202 4969-02-04 09:30:55.202 4964-10-04 10:30:55.202 4964-10-04 10:30:55.202 4969-02-04 09:30:55.202
+5339-02-01 14:10:01.085678691 5336-12-01 14:10:01.085678691 5341-04-01 15:10:01.085678691 5341-04-01 15:10:01.085678691 5336-12-01 14:10:01.085678691 5336-12-01 14:10:01.085678691 5341-04-01 15:10:01.085678691
+5344-10-04 18:40:08.165 5342-08-04 18:40:08.165 5346-12-04 17:40:08.165 5346-12-04 17:40:08.165 5342-08-04 18:40:08.165 5342-08-04 18:40:08.165 5346-12-04 17:40:08.165
+5397-07-13 07:12:32.000896438 5395-05-13 07:12:32.000896438 5399-09-13 07:12:32.000896438 5399-09-13 07:12:32.000896438 5395-05-13 07:12:32.000896438 5395-05-13 07:12:32.000896438 5399-09-13 07:12:32.000896438
+5966-07-09 03:30:50.597 5964-05-09 03:30:50.597 5968-09-09 03:30:50.597 5968-09-09 03:30:50.597 5964-05-09 03:30:50.597 5964-05-09 03:30:50.597 5968-09-09 03:30:50.597
+6229-06-28 02:54:28.970117179 6227-04-28 02:54:28.970117179 6231-08-28 02:54:28.970117179 6231-08-28 02:54:28.970117179 6227-04-28 02:54:28.970117179 6227-04-28 02:54:28.970117179 6231-08-28 02:54:28.970117179
+6482-04-27 12:07:38.073915413 6480-02-27 11:07:38.073915413 6484-06-27 12:07:38.073915413 6484-06-27 12:07:38.073915413 6480-02-27 11:07:38.073915413 6480-02-27 11:07:38.073915413 6484-06-27 12:07:38.073915413
+6631-11-13 16:31:29.702202248 6629-09-13 17:31:29.702202248 6634-01-13 16:31:29.702202248 6634-01-13 16:31:29.702202248 6629-09-13 17:31:29.702202248 6629-09-13 17:31:29.702202248 6634-01-13 16:31:29.702202248
+6705-09-28 18:27:28.000845672 6703-07-28 18:27:28.000845672 6707-11-28 17:27:28.000845672 6707-11-28 17:27:28.000845672 6703-07-28 18:27:28.000845672 6703-07-28 18:27:28.000845672 6707-11-28 17:27:28.000845672
+6731-02-12 08:12:48.287783702 6728-12-12 08:12:48.287783702 6733-04-12 09:12:48.287783702 6733-04-12 09:12:48.287783702 6728-12-12 08:12:48.287783702 6728-12-12 08:12:48.287783702 6733-04-12 09:12:48.287783702
+7160-12-02 06:00:24.81200852 7158-10-02 07:00:24.81200852 7163-02-02 06:00:24.81200852 7163-02-02 06:00:24.81200852 7158-10-02 07:00:24.81200852 7158-10-02 07:00:24.81200852 7163-02-02 06:00:24.81200852
+7409-09-07 23:33:32.459349602 7407-07-07 23:33:32.459349602 7411-11-07 22:33:32.459349602 7411-11-07 22:33:32.459349602 7407-07-07 23:33:32.459349602 7407-07-07 23:33:32.459349602 7411-11-07 22:33:32.459349602
+7503-06-23 23:14:17.486 7501-04-23 23:14:17.486 7505-08-23 23:14:17.486 7505-08-23 23:14:17.486 7501-04-23 23:14:17.486 7501-04-23 23:14:17.486 7505-08-23 23:14:17.486
+8422-07-22 03:21:45.745036084 8420-05-22 03:21:45.745036084 8424-09-22 03:21:45.745036084 8424-09-22 03:21:45.745036084 8420-05-22 03:21:45.745036084 8420-05-22 03:21:45.745036084 8424-09-22 03:21:45.745036084
+8521-01-16 20:42:05.668832388 8518-11-16 20:42:05.668832388 8523-03-16 21:42:05.668832388 8523-03-16 21:42:05.668832388 8518-11-16 20:42:05.668832388 8518-11-16 20:42:05.668832388 8523-03-16 21:42:05.668832388
+9075-06-13 16:20:09.218517797 9073-04-13 16:20:09.218517797 9077-08-13 16:20:09.218517797 9077-08-13 16:20:09.218517797 9073-04-13 16:20:09.218517797 9073-04-13 16:20:09.218517797 9077-08-13 16:20:09.218517797
+9209-11-11 04:08:58.223768453 9207-09-11 05:08:58.223768453 9212-01-11 04:08:58.223768453 9212-01-11 04:08:58.223768453 9207-09-11 05:08:58.223768453 9207-09-11 05:08:58.223768453 9212-01-11 04:08:58.223768453
+9403-01-09 18:12:33.547 9400-11-09 18:12:33.547 9405-03-09 18:12:33.547 9405-03-09 18:12:33.547 9400-11-09 18:12:33.547 9400-11-09 18:12:33.547 9405-03-09 18:12:33.547
+PREHOOK: query: explain
+select
+ interval '2-2' year to month + interval '3-3' year to month,
+ interval '2-2' year to month - interval '3-3' year to month
+from interval_arithmetic_1
+order by interval '2-2' year to month + interval '3-3' year to month
+limit 2
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select
+ interval '2-2' year to month + interval '3-3' year to month,
+ interval '2-2' year to month - interval '3-3' year to month
+from interval_arithmetic_1
+order by interval '2-2' year to month + interval '3-3' year to month
+limit 2
+POSTHOOK: type: QUERY
+Explain
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: interval_arithmetic_1
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ Statistics: Num rows: 50 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: 5-5 (type: interval_year_month)
+ sort order: +
+ Statistics: Num rows: 50 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ TopN Hash Memory Usage: 0.1
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Select Operator
+ expressions: 5-5 (type: interval_year_month), -1-1 (type: interval_year_month)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 50 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Limit
+ Number of rows: 2
+ Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 2
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select
+ interval '2-2' year to month + interval '3-3' year to month,
+ interval '2-2' year to month - interval '3-3' year to month
+from interval_arithmetic_1
+order by interval '2-2' year to month + interval '3-3' year to month
+limit 2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@interval_arithmetic_1
+#### A masked pattern was here ####
+POSTHOOK: query: select
+ interval '2-2' year to month + interval '3-3' year to month,
+ interval '2-2' year to month - interval '3-3' year to month
+from interval_arithmetic_1
+order by interval '2-2' year to month + interval '3-3' year to month
+limit 2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@interval_arithmetic_1
+#### A masked pattern was here ####
+c0 c1
+5-5 -1-1
+5-5 -1-1
+PREHOOK: query: -- interval day-time arithmetic
+explain
+select
+ dateval,
+ dateval - interval '99 11:22:33.123456789' day to second,
+ dateval - interval '-99 11:22:33.123456789' day to second,
+ dateval + interval '99 11:22:33.123456789' day to second,
+ dateval + interval '-99 11:22:33.123456789' day to second,
+ -interval '99 11:22:33.123456789' day to second + dateval,
+ interval '99 11:22:33.123456789' day to second + dateval
+from interval_arithmetic_1
+order by dateval
+PREHOOK: type: QUERY
+POSTHOOK: query: -- interval day-time arithmetic
+explain
+select
+ dateval,
+ dateval - interval '99 11:22:33.123456789' day to second,
+ dateval - interval '-99 11:22:33.123456789' day to second,
+ dateval + interval '99 11:22:33.123456789' day to second,
+ dateval + interval '-99 11:22:33.123456789' day to second,
+ -interval '99 11:22:33.123456789' day to second + dateval,
+ interval '99 11:22:33.123456789' day to second + dateval
+from interval_arithmetic_1
+order by dateval
+POSTHOOK: type: QUERY
+Explain
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: interval_arithmetic_1
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: dateval (type: date), (dateval - 99 11:22:33.123456789) (type: timestamp), (dateval - -99 11:22:33.123456789) (type: timestamp), (dateval + 99 11:22:33.123456789) (type: timestamp), (dateval + -99 11:22:33.123456789) (type: timestamp), (-99 11:22:33.123456789 + dateval) (type: timestamp), (99 11:22:33.123456789 + dateval) (type: timestamp)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: date)
+ sort order: +
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: timestamp), _col2 (type: timestamp), _col3 (type: timestamp), _col4 (type: timestamp), _col5 (type: timestamp), _col6 (type: timestamp)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: date), VALUE._col0 (type: timestamp), VALUE._col1 (type: timestamp), VALUE._col2 (type: timestamp), VALUE._col3 (type: timestamp), VALUE._col4 (type: timestamp), VALUE._col5 (type: timestamp)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select
+ dateval,
+ dateval - interval '99 11:22:33.123456789' day to second,
+ dateval - interval '-99 11:22:33.123456789' day to second,
+ dateval + interval '99 11:22:33.123456789' day to second,
+ dateval + interval '-99 11:22:33.123456789' day to second,
+ -interval '99 11:22:33.123456789' day to second + dateval,
+ interval '99 11:22:33.123456789' day to second + dateval
+from interval_arithmetic_1
+order by dateval
+PREHOOK: type: QUERY
+PREHOOK: Input: default@interval_arithmetic_1
+#### A masked pattern was here ####
+POSTHOOK: query: select
+ dateval,
+ dateval - interval '99 11:22:33.123456789' day to second,
+ dateval - interval '-99 11:22:33.123456789' day to second,
+ dateval + interval '99 11:22:33.123456789' day to second,
+ dateval + interval '-99 11:22:33.123456789' day to second,
+ -interval '99 11:22:33.123456789' day to second + dateval,
+ interval '99 11:22:33.123456789' day to second + dateval
+from interval_arithmetic_1
+order by dateval
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@interval_arithmetic_1
+#### A masked pattern was here ####
+dateval _c1 _c2 _c3 _c4 _c5 _c6
+0004-09-22 0004-06-14 12:37:26.876543211 0004-12-30 11:22:33.123456789 0004-12-30 11:22:33.123456789 0004-06-14 12:37:26.876543211 0004-06-14 12:37:26.876543211 0004-12-30 11:22:33.123456789
+0528-10-27 0528-07-19 12:37:26.876543211 0529-02-03 11:22:33.123456789 0529-02-03 11:22:33.123456789 0528-07-19 12:37:26.876543211 0528-07-19 12:37:26.876543211 0529-02-03 11:22:33.123456789
+1319-02-02 1318-10-25 12:37:26.876543211 1319-05-12 11:22:33.123456789 1319-05-12 11:22:33.123456789 1318-10-25 12:37:26.876543211 1318-10-25 12:37:26.876543211 1319-05-12 11:22:33.123456789
+1404-07-23 1404-04-14 12:37:26.876543211 1404-10-30 11:22:33.123456789 1404-10-30 11:22:33.123456789 1404-04-14 12:37:26.876543211 1404-04-14 12:37:26.876543211 1404-10-30 11:22:33.123456789
+1815-05-06 1815-01-26 12:37:26.876543211 1815-08-13 11:22:33.123456789 1815-08-13 11:22:33.123456789 1815-01-26 12:37:26.876543211 1815-01-26 12:37:26.876543211 1815-08-13 11:22:33.123456789
+1883-04-17 1883-01-07 12:37:26.876543211 1883-07-25 11:22:33.123456789 1883-07-25 11:22:33.123456789 1883-01-07 12:37:26.876543211 1883-01-07 12:37:26.876543211 1883-07-25 11:22:33.123456789
+1966-08-16 1966-05-08 12:37:26.876543211 1966-11-23 10:22:33.123456789 1966-11-23 10:22:33.123456789 1966-05-08 12:37:26.876543211 1966-05-08 12:37:26.876543211 1966-11-23 10:22:33.123456789
+1973-04-17 1973-01-07 12:37:26.876543211 1973-07-25 12:22:33.123456789 1973-07-25 12:22:33.123456789 1973-01-07 12:37:26.876543211 1973-01-07 12:37:26.876543211 1973-07-25 12:22:33.123456789
+1974-10-04 1974-06-26 12:37:26.876543211 1975-01-11 10:22:33.123456789 1975-01-11 10:22:33.123456789 1974-06-26 12:37:26.876543211 1974-06-26 12:37:26.876543211 1975-01-11 10:22:33.123456789
+1976-03-03 1975-11-24 12:37:26.876543211 1976-06-10 12:22:33.123456789 1976-06-10 12:22:33.123456789 1975-11-24 12:37:26.876543211 1975-11-24 12:37:26.876543211 1976-06-10 12:22:33.123456789
+1976-05-06 1976-01-27 11:37:26.876543211 1976-08-13 11:22:33.123456789 1976-08-13 11:22:33.123456789 1976-01-27 11:37:26.876543211 1976-01-27 11:37:26.876543211 1976-08-13 11:22:33.123456789
+1978-08-05 1978-04-27 11:37:26.876543211 1978-11-12 10:22:33.123456789 1978-11-12 10:22:33.123456789 1978-04-27 11:37:26.876543211 1978-04-27 11:37:26.876543211 1978-11-12 10:22:33.123456789
+1981-04-25 1981-01-15 12:37:26.876543211 1981-08-02 12:22:33.123456789 1981-08-02 12:22:33.123456789 1981-01-15 12:37:26.876543211 1981-01-15 12:37:26.876543211 1981-08-02 12:22:33.123456789
+1981-11-15 1981-08-07 13:37:26.876543211 1982-02-22 11:22:33.123456789 1982-02-22 11:22:33.123456789 1981-08-07 13:37:26.876543211 1981-08-07 13:37:26.876543211 1982-02-22 11:22:33.123456789
+1985-07-20 1985-04-11 11:37:26.876543211 1985-10-27 10:22:33.123456789 1985-10-27 10:22:33.123456789 1985-04-11 11:37:26.876543211 1985-04-11 11:37:26.876543211 1985-10-27 10:22:33.123456789
+1985-11-18 1985-08-10 13:37:26.876543211 1986-02-25 11:22:33.123456789 1986-02-25 11:22:33.123456789 1985-08-10 13:37:26.876543211 1985-08-10 13:37:26.876543211 1986-02-25 11:22:33.123456789
+1987-02-21 1986-11-13 12:37:26.876543211 1987-05-31 12:22:33.123456789 1987-05-31 12:22:33.123456789 1986-11-13 12:37:26.876543211 1986-11-13 12:37:26.876543211 1987-05-31 12:22:33.123456789
+1987-05-28 1987-02-17 11:37:26.876543211 1987-09-04 11:22:33.123456789 1987-09-04 11:22:33.123456789 1987-02-17 11:37:26.876543211 1987-02-17 11:37:26.876543211 1987-09-04 11:22:33.123456789
+1998-10-16 1998-07-08 12:37:26.876543211 1999-01-23 10:22:33.123456789 1999-01-23 10:22:33.123456789 1998-07-08 12:37:26.876543211 1998-07-08 12:37:26.876543211 1999-01-23 10:22:33.123456789
+1999-10-03 1999-06-25 12:37:26.876543211 2000-01-10 10:22:33.123456789 2000-01-10 10:22:33.123456789 1999-06-25 12:37:26.876543211 1999-06-25 12:37:26.876543211 2000-01-10 10:22:33.123456789
+2000-12-18 2000-09-09 13:37:26.876543211 2001-03-27 11:22:33.123456789 2001-03-27 11:22:33.123456789 2000-09-09 13:37:26.876543211 2000-09-09 13:37:26.876543211 2001-03-27 11:22:33.123456789
+2002-05-10 2002-01-30 11:37:26.876543211 2002-08-17 11:22:33.123456789 2002-08-17 11:22:33.123456789 2002-01-30 11:37:26.876543211 2002-01-30 11:37:26.876543211 2002-08-17 11:22:33.123456789
+2003-09-23 2003-06-15 12:37:26.876543211 2003-12-31 10:22:33.123456789 2003-12-31 10:22:33.123456789 2003-06-15 12:37:26.876543211 2003-06-15 12:37:26.876543211 2003-12-31 10:22:33.123456789
+2004-03-07 2003-11-28 12:37:26.876543211 2004-06-14 12:22:33.123456789 2004-06-14 12:22:33.123456789 2003-11-28 12:37:26.876543211 2003-11-28 12:37:26.876543211 2004-06-14 12:22:33.123456789
+2007-02-09 2006-11-01 12:37:26.876543211 2007-05-19 12:22:33.123456789 2007-05-19 12:22:33.123456789 2006-11-01 12:37:26.876543211 2006-11-01 12:37:26.876543211 2007-05-19 12:22:33.123456789
+2009-01-21 2008-10-13 13:37:26.876543211 2009-04-30 12:22:33.123456789 2009-04-30 12:22:33.123456789 2008-10-13 13:37:26.876543211 2008-10-13 13:37:26.876543211 2009-04-30 12:22:33.123456789
+2010-04-08 2009-12-29 11:37:26.876543211 2010-07-16 11:22:33.123456789 2010-07-16 11:22:33.123456789 2009-12-29 11:37:26.876543211 2009-12-29 11:37:26.876543211 2010-07-16 11:22:33.123456789
+2013-04-07 2012-12-28 11:37:26.876543211 2013-07-15 11:22:33.123456789 2013-07-15 11:22:33.123456789 2012-12-28 11:37:26.876543211 2012-12-28 11:37:26.876543211 2013-07-15 11:22:33.123456789
+2013-04-10 2012-12-31 11:37:26.876543211 2013-07-18 11:22:33.123456789 2013-07-18 11:22:33.123456789 2012-12-31 11:37:26.876543211 2012-12-31 11:37:26.876543211 2013-07-18 11:22:33.123456789
+2021-09-24 2021-06-16 12:37:26.876543211 2022-01-01 10:22:33.123456789 2022-01-01 10:22:33.123456789 2021-06-16 12:37:26.876543211 2021-06-16 12:37:26.876543211 2022-01-01 10:22:33.123456789
+2024-11-11 2024-08-03 13:37:26.876543211 2025-02-18 11:22:33.123456789 2025-02-18 11:22:33.123456789 2024-08-03 13:37:26.876543211 2024-08-03 13:37:26.876543211 2025-02-18 11:22:33.123456789
+4143-07-08 4143-03-30 12:37:26.876543211 4143-10-15 11:22:33.123456789 4143-10-15 11:22:33.123456789 4143-03-30 12:37:26.876543211 4143-03-30 12:37:26.876543211 4143-10-15 11:22:33.123456789
+4966-12-04 4966-08-26 13:37:26.876543211 4967-03-13 12:22:33.123456789 4967-03-13 12:22:33.123456789 4966-08-26 13:37:26.876543211 4966-08-26 13:37:26.876543211 4967-03-13 12:22:33.123456789
+5339-02-01 5338-10-24 13:37:26.876543211 5339-05-11 12:22:33.123456789 5339-05-11 12:22:33.123456789 5338-10-24 13:37:26.876543211 5338-10-24 13:37:26.876543211 5339-05-11 12:22:33.123456789
+5344-10-04 5344-06-26 12:37:26.876543211 5345-01-11 10:22:33.123456789 5345-01-11 10:22:33.123456789 5344-06-26 12:37:26.876543211 5344-06-26 12:37:26.876543211 5345-01-11 10:22:33.123456789
+5397-07-13 5397-04-04 12:37:26.876543211 5397-10-20 11:22:33.123456789 5397-10-20 11:22:33.123456789 5397-04-04 12:37:26.876543211 5397-04-04 12:37:26.876543211 5397-10-20 11:22:33.123456789
+5966-07-09 5966-03-31 12:37:26.876543211 5966-10-16 11:22:33.123456789 5966-10-16 11:22:33.123456789 5966-03-31 12:37:26.876543211 5966-03-31 12:37:26.876543211 5966-10-16 11:22:33.123456789
+6229-06-28 6229-03-20 12:37:26.876543211 6229-10-05 11:22:33.123456789 6229-10-05 11:22:33.123456789 6229-03-20 12:37:26.876543211 6229-03-20 12:37:26.876543211 6229-10-05 11:22:33.123456789
+6482-04-27 6482-01-17 11:37:26.876543211 6482-08-04 11:22:33.123456789 6482-08-04 11:22:33.123456789 6482-01-17 11:37:26.876543211 6482-01-17 11:37:26.876543211 6482-08-04 11:22:33.123456789
+6631-11-13 6631-08-05 13:37:26.876543211 6632-02-20 11:22:33.123456789 6632-02-20 11:22:33.123456789 6631-08-05 13:37:26.876543211 6631-08-05 13:37:26.876543211 6632-02-20 11:22:33.123456789
+6705-09-28 6705-06-20 12:37:26.876543211 6706-01-05 10:22:33.123456789 6706-01-05 10:22:33.123456789 6705-06-20 12:37:26.876543211 6705-06-20 12:37:26.876543211 6706-01-05 10:22:33.123456789
+6731-02-12 6730-11-04 12:37:26.876543211 6731-05-22 12:22:33.123456789 6731-05-22 12:22:33.123456789 6730-11-04 12:37:26.876543211 6730-11-04 12:37:26.876543211 6731-05-22 12:22:33.123456789
+7160-12-02 7160-08-24 13:37:26.876543211 7161-03-11 11:22:33.123456789 7161-03-11 11:22:33.123456789 7160-08-24 13:37:26.876543211 7160-08-24 13:37:26.876543211 7161-03-11 11:22:33.123456789
+7409-09-07 7409-05-30 12:37:26.876543211 7409-12-15 10:22:33.123456789 7409-12-15 10:22:33.123456789 7409-05-30 12:37:26.876543211 7409-05-30 12:37:26.876543211 7409-12-15 10:22:33.123456789
+7503-06-23 7503-03-15 12:37:26.876543211 7503-09-30 11:22:33.123456789 7503-09-30 11:22:33.123456789 7503-03-15 12:37:26.876543211 7503-03-15 12:37:26.876543211 7503-09-30 11:22:33.123456789
+8422-07-22 8422-04-13 12:37:26.876543211 8422-10-29 11:22:33.123456789 8422-10-29 11:22:33.123456789 8422-04-13 12:37:26.876543211 8422-04-13 12:37:26.876543211 8422-10-29 11:22:33.123456789
+8521-01-16 8520-10-08 13:37:26.876543211 8521-04-25 12:22:33.123456789 8521-04-25 12:22:33.123456789 8520-10-08 13:37:26.876543211 8520-10-08 13:37:26.876543211 8521-04-25 12:22:33.123456789
+9075-06-13 9075-03-05 11:37:26.876543211 9075-09-20 11:22:33.123456789 9075-09-20 11:22:33.123456789 9075-03-05 11:37:26.876543211 9075-03-05 11:37:26.876543211 9075-09-20 11:22:33.123456789
+9209-11-11 9209-08-03 13:37:26.876543211 9210-02-18 11:22:33.123456789 9210-02-18 11:22:33.123456789 9209-08-03 13:37:26.876543211 9209-08-03 13:37:26.876543211 9210-02-18 11:22:33.123456789
+9403-01-09 9402-10-01 13:37:26.876543211 9403-04-18 12:22:33.123456789 9403-04-18 12:22:33.123456789 9402-10-01 13:37:26.876543211 9402-10-01 13:37:26.876543211 9403-04-18 12:22:33.123456789
+PREHOOK: query: explain
+select
+ dateval,
+ tsval,
+ dateval - tsval,
+ tsval - dateval,
+ tsval - tsval
+from interval_arithmetic_1
+order by dateval
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select
+ dateval,
+ tsval,
+ dateval - tsval,
+ tsval - dateval,
+ tsval - tsval
+from interval_arithmetic_1
+order by dateval
+POSTHOOK: type: QUERY
+Explain
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: interval_arithmetic_1
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: dateval (type: date), tsval (type: timestamp), (dateval - tsval) (type: interval_day_time), (tsval - dateval) (type: interval_day_time), (tsval - tsval) (type: interval_day_time)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: date)
+ sort order: +
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: timestamp), _col2 (type: interval_day_time), _col3 (type: interval_day_time), _col4 (type: interval_day_time)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: date), VALUE._col0 (type: timestamp), VALUE._col1 (type: interval_day_time), VALUE._col2 (type: interval_day_time), VALUE._col3 (type: interval_day_time)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select
+ dateval,
+ tsval,
+ dateval - tsval,
+ tsval - dateval,
+ tsval - tsval
+from interval_arithmetic_1
+order by dateval
+PREHOOK: type: QUERY
+PREHOOK: Input: default@interval_arithmetic_1
+#### A masked pattern was here ####
+POSTHOOK: query: select
+ dateval,
+ tsval,
+ dateval - tsval,
+ tsval - dateval,
+ tsval - tsval
+from interval_arithmetic_1
+order by dateval
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@interval_arithmetic_1
+#### A masked pattern was here ####
+dateval tsval c2 c3 c4
+0004-09-22 0004-09-22 18:26:29.519542222 -0 18:26:30.519542222 0 18:26:30.519542222 0 00:00:00.000000000
+0528-10-27 0528-10-27 08:15:18.941718273 -0 08:15:19.941718273 0 08:15:19.941718273 0 00:00:00.000000000
+1319-02-02 1319-02-02 16:31:57.778 -0 16:31:58.778000000 0 16:31:58.778000000 0 00:00:00.000000000
+1404-07-23 1404-07-23 15:32:16.059185026 -0 15:32:17.059185026 0 15:32:17.059185026 0 00:00:00.000000000
+1815-05-06 1815-05-06 00:12:37.543584705 -0 00:12:38.543584705 0 00:12:38.543584705 0 00:00:00.000000000
+1883-04-17 1883-04-17 04:14:34.647766229 -0 04:14:35.647766229 0 04:14:35.647766229 0 00:00:00.000000000
+1966-08-16 1966-08-16 13:36:50.183618031 -0 13:36:51.183618031 0 13:36:51.183618031 0 00:00:00.000000000
+1973-04-17 1973-04-17 06:30:38.596784156 -0 06:30:38.596784156 0 06:30:38.596784156 0 00:00:00.000000000
+1974-10-04 1974-10-04 17:21:03.989 -0 17:21:03.989000000 0 17:21:03.989000000 0 00:00:00.000000000
+1976-03-03 1976-03-03 04:54:33.000895162 -0 04:54:33.000895162 0 04:54:33.000895162 0 00:00:00.000000000
+1976-05-06 1976-05-06 00:42:30.910786948 -0 00:42:30.910786948 0 00:42:30.910786948 0 00:00:00.000000000
+1978-08-05 1978-08-05 14:41:05.501 -0 14:41:05.501000000 0 14:41:05.501000000 0 00:00:00.000000000
+1981-04-25 1981-04-25 09:01:12.077192689 -0 09:01:12.077192689 0 09:01:12.077192689 0 00:00:00.000000000
+1981-11-15 1981-11-15 23:03:10.999338387 -0 23:03:10.999338387 0 23:03:10.999338387 0 00:00:00.000000000
+1985-07-20 1985-07-20 09:30:11 -0 09:30:11.000000000 0 09:30:11.000000000 0 00:00:00.000000000
+1985-11-18 1985-11-18 16:37:54 -0 16:37:54.000000000 0 16:37:54.000000000 0 00:00:00.000000000
+1987-02-21 1987-02-21 19:48:29 -0 19:48:29.000000000 0 19:48:29.000000000 0 00:00:00.000000000
+1987-05-28 1987-05-28 13:52:07.900916635 -0 13:52:07.900916635 0 13:52:07.900916635 0 00:00:00.000000000
+1998-10-16 1998-10-16 20:05:29.397591987 -0 20:05:29.397591987 0 20:05:29.397591987 0 00:00:00.000000000
+1999-10-03 1999-10-03 16:59:10.396903939 -0 16:59:10.396903939 0 16:59:10.396903939 0 00:00:00.000000000
+2000-12-18 2000-12-18 08:42:30.000595596 -0 08:42:30.000595596 0 08:42:30.000595596 0 00:00:00.000000000
+2002-05-10 2002-05-10 05:29:48.990818073 -0 05:29:48.990818073 0 05:29:48.990818073 0 00:00:00.000000000
+2003-09-23 2003-09-23 22:33:17.00003252 -0 22:33:17.000032520 0 22:33:17.000032520 0 00:00:00.000000000
+2004-03-07 2004-03-07 20:14:13 -0 20:14:13.000000000 0 20:14:13.000000000 0 00:00:00.000000000
+2007-02-09 2007-02-09 05:17:29.368756876 -0 05:17:29.368756876 0 05:17:29.368756876 0 00:00:00.000000000
+2009-01-21 2009-01-21 10:49:07.108 -0 10:49:07.108000000 0 10:49:07.108000000 0 00:00:00.000000000
+2010-04-08 2010-04-08 02:43:35.861742727 -0 02:43:35.861742727 0 02:43:35.861742727 0 00:00:00.000000000
+2013-04-07 2013-04-07 02:44:43.00086821 -0 02:44:43.000868210 0 02:44:43.000868210 0 00:00:00.000000000
+2013-04-10 2013-04-10 00:43:46.854731546 -0 00:43:46.854731546 0 00:43:46.854731546 0 00:00:00.000000000
+2021-09-24 2021-09-24 03:18:32.413655165 -0 03:18:32.413655165 0 03:18:32.413655165 0 00:00:00.000000000
+2024-11-11 2024-11-11 16:42:41.101 -0 16:42:41.101000000 0 16:42:41.101000000 0 00:00:00.000000000
+4143-07-08 4143-07-08 10:53:27.252802259 -0 10:53:27.252802259 0 10:53:27.252802259 0 00:00:00.000000000
+4966-12-04 4966-12-04 09:30:55.202 -0 09:30:55.202000000 0 09:30:55.202000000 0 00:00:00.000000000
+5339-02-01 5339-02-01 14:10:01.085678691 -0 14:10:01.085678691 0 14:10:01.085678691 0 00:00:00.000000000
+5344-10-04 5344-10-04 18:40:08.165 -0 18:40:08.165000000 0 18:40:08.165000000 0 00:00:00.000000000
+5397-07-13 5397-07-13 07:12:32.000896438 -0 07:12:32.000896438 0 07:12:32.000896438 0 00:00:00.000000000
+5966-07-09 5966-07-09 03:30:50.597 -0 03:30:50.597000000 0 03:30:50.597000000 0 00:00:00.000000000
+6229-06-28 6229-06-28 02:54:28.970117179 -0 02:54:28.970117179 0 02:54:28.970117179 0 00:00:00.000000000
+6482-04-27 6482-04-27 12:07:38.073915413 -0 12:07:38.073915413 0 12:07:38.073915413 0 00:00:00.000000000
+6631-11-13 6631-11-13 16:31:29.702202248 -0 16:31:29.702202248 0 16:31:29.702202248 0 00:00:00.000000000
+6705-09-28 6705-09-28 18:27:28.000845672 -0 18:27:28.000845672 0 18:27:28.000845672 0 00:00:00.000000000
+6731-02-12 6731-02-12 08:12:48.287783702 -0 08:12:48.287783702 0 08:12:48.287783702 0 00:00:00.000000000
+7160-12-02 7160-12-02 06:00:24.81200852 -0 06:00:24.812008520 0 06:00:24.812008520 0 00:00:00.000000000
+7409-09-07 7409-09-07 23:33:32.459349602 -0 23:33:32.459349602 0 23:33:32.459349602 0 00:00:00.000000000
+7503-06-23 7503-06-23 23:14:17.486 -0 23:14:17.486000000 0 23:14:17.486000000 0 00:00:00.000000000
+8422-07-22 8422-07-22 03:21:45.745036084 -0 03:21:45.745036084 0 03:21:45.745036084 0 00:00:00.000000000
+8521-01-16 8521-01-16 20:42:05.668832388 -0 20:42:05.668832388 0 20:42:05.668832388 0 00:00:00.000000000
+9075-06-13 9075-06-13 16:20:09.218517797 -0 16:20:09.218517797 0 16:20:09.218517797 0 00:00:00.000000000
+9209-11-11 9209-11-11 04:08:58.223768453 -0 04:08:58.223768453 0 04:08:58.223768453 0 00:00:00.000000000
+9403-01-09 9403-01-09 18:12:33.547 -0 18:12:33.547000000 0 18:12:33.547000000 0 00:00:00.000000000
+PREHOOK: query: explain
+select
+ tsval,
+ tsval - interval '99 11:22:33.123456789' day to second,
+ tsval - interval '-99 11:22:33.123456789' day to second,
+ tsval + interval '99 11:22:33.123456789' day to second,
+ tsval + interval '-99 11:22:33.123456789' day to second,
+ -interval '99 11:22:33.123456789' day to second + tsval,
+ interval '99 11:22:33.123456789' day to second + tsval
+from interval_arithmetic_1
+order by tsval
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select
+ tsval,
+ tsval - interval '99 11:22:33.123456789' day to second,
+ tsval - interval '-99 11:22:33.123456789' day to second,
+ tsval + interval '99 11:22:33.123456789' day to second,
+ tsval + interval '-99 11:22:33.123456789' day to second,
+ -interval '99 11:22:33.123456789' day to second + tsval,
+ interval '99 11:22:33.123456789' day to second + tsval
+from interval_arithmetic_1
+order by tsval
+POSTHOOK: type: QUERY
+Explain
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: interval_arithmetic_1
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: tsval (type: timestamp), (tsval - 99 11:22:33.123456789) (type: timestamp), (tsval - -99 11:22:33.123456789) (type: timestamp), (tsval + 99 11:22:33.123456789) (type: timestamp), (tsval + -99 11:22:33.123456789) (type: timestamp), (-99 11:22:33.123456789 + tsval) (type: timestamp), (99 11:22:33.123456789 + tsval) (type: timestamp)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: timestamp)
+ sort order: +
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: timestamp), _col2 (type: timestamp), _col3 (type: timestamp), _col4 (type: timestamp), _col5 (type: timestamp), _col6 (type: timestamp)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: timestamp), VALUE._col0 (type: timestamp), VALUE._col1 (type: timestamp), VALUE._col2 (type: timestamp), VALUE._col3 (type: timestamp), VALUE._col4 (type: timestamp), VALUE._col5 (type: timestamp)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select
+ tsval,
+ tsval - interval '99 11:22:33.123456789' day to second,
+ tsval - interval '-99 11:22:33.123456789' day to second,
+ tsval + interval '99 11:22:33.123456789' day to second,
+ tsval + interval '-99 11:22:33.123456789' day to second,
+ -interval '99 11:22:33.123456789' day to second + tsval,
+ interval '99 11:22:33.123456789' day to second + tsval
+from interval_arithmetic_1
+order by tsval
+PREHOOK: type: QUERY
+PREHOOK: Input: default@interval_arithmetic_1
+#### A masked pattern was here ####
+POSTHOOK: query: select
+ tsval,
+ tsval - interval '99 11:22:33.123456789' day to second,
+ tsval - interval '-99 11:22:33.123456789' day to second,
+ tsval + interval '99 11:22:33.123456789' day to second,
+ tsval + interval '-99 11:22:33.123456789' day to second,
+ -interval '99 11:22:33.123456789' day to second + tsval,
+ interval '99 11:22:33.123456789' day to second + tsval
+from interval_arithmetic_1
+order by tsval
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@interval_arithmetic_1
+#### A masked pattern was here ####
+tsval _c1 _c2 _c3 _c4 _c5 _c6
+0004-09-22 18:26:29.519542222 0004-06-15 07:03:56.396085433 0004-12-31 05:49:02.642999011 0004-12-31 05:49:02.642999011 0004-06-15 07:03:56.396085433 0004-06-15 07:03:56.396085433 0004-12-31 05:49:02.642999011
+0528-10-27 08:15:18.941718273 0528-07-19 20:52:45.818261484 0529-02-03 19:37:52.065175062 0529-02-03 19:37:52.065175062 0528-07-19 20:52:45.818261484 0528-07-19 20:52:45.818261484 0529-02-03 19:37:52.065175062
+1319-02-02 16:31:57.778 1318-10-26 05:09:24.654543211 1319-05-13 03:54:30.901456789 1319-05-13 03:54:30.901456789 1318-10-26 05:09:24.654543211 1318-10-26 05:09:24.654543211 1319-05-13 03:54:30.901456789
+1404-07-23 15:32:16.059185026 1404-04-15 04:09:42.935728237 1404-10-31 02:54:49.182641815 1404-10-31 02:54:49.182641815 1404-04-15 04:09:42.935728237 1404-04-15 04:09:42.935728237 1404-10-31 02:54:49.182641815
+1815-05-06 00:12:37.543584705 1815-01-26 12:50:04.420127916 1815-08-13 11:35:10.667041494 1815-08-13 11:35:10.667041494 1815-01-26 12:50:04.420127916 1815-01-26 12:50:04.420127916 1815-08-13 11:35:10.667041494
+1883-04-17 04:14:34.647766229 1883-01-07 16:52:01.52430944 1883-07-25 15:37:07.771223018 1883-07-25 15:37:07.771223018 1883-01-07 16:52:01.52430944 1883-01-07 16:52:01.52430944 1883-07-25 15:37:07.771223018
+1966-08-16 13:36:50.183618031 1966-05-09 02:14:17.060161242 1966-11-23 23:59:23.30707482 1966-11-23 23:59:23.30707482 1966-05-09 02:14:17.060161242 1966-05-09 02:14:17.060161242 1966-11-23 23:59:23.30707482
+1973-04-17 06:30:38.596784156 1973-01-07 19:08:05.473327367 1973-07-25 18:53:11.720240945 1973-07-25 18:53:11.720240945 1973-01-07 19:08:05.473327367 1973-01-07 19:08:05.473327367 1973-07-25 18:53:11.720240945
+1974-10-04 17:21:03.989 1974-06-27 05:58:30.865543211 1975-01-12 03:43:37.112456789 1975-01-12 03:43:37.112456789 1974-06-27 05:58:30.865543211 1974-06-27 05:58:30.865543211 1975-01-12 03:43:37.112456789
+1976-03-03 04:54:33.000895162 1975-11-24 17:31:59.877438373 1976-06-10 17:17:06.124351951 1976-06-10 17:17:06.124351951 1975-11-24 17:31:59.877438373 1975-11-24 17:31:59.877438373 1976-06-10 17:17:06.124351951
+1976-05-06 00:42:30.910786948 1976-01-27 12:19:57.787330159 1976-08-13 12:05:04.034243737 1976-08-13 12:05:04.034243737 1976-01-27 12:19:57.787330159 1976-01-27 12:19:57.787330159 1976-08-13 12:05:04.034243737
+1978-08-05 14:41:05.501 1978-04-28 02:18:32.377543211 1978-11-13 01:03:38.624456789 1978-11-13 01:03:38.624456789 1978-04-28 02:18:32.377543211 1978-04-28 02:18:32.377543211 1978-11-13 01:03:38.624456789
+1981-04-25 09:01:12.077192689 1981-01-15 21:38:38.9537359 1981-08-02 21:23:45.200649478 1981-08-02 21:23:45.200649478 1981-01-15 21:38:38.9537359 1981-01-15 21:38:38.9537359 1981-08-02 21:23:45.200649478
+1981-11-15 23:03:10.999338387 1981-08-08 12:40:37.875881598 1982-02-23 10:25:44.122795176 1982-02-23 10:25:44.122795176 1981-08-08 12:40:37.875881598 1981-08-08 12:40:37.875881598 1982-02-23 10:25:44.122795176
+1985-07-20 09:30:11 1985-04-11 21:07:37.876543211 1985-10-27 19:52:44.123456789 1985-10-27 19:52:44.123456789 1985-04-11 21:07:37.876543211 1985-04-11 21:07:37.876543211 1985-10-27 19:52:44.123456789
+1985-11-18 16:37:54 1985-08-11 06:15:20.876543211 1986-02-26 04:00:27.123456789 1986-02-26 04:00:27.123456789 1985-08-11 06:15:20.876543211 1985-08-11 06:15:20.876543211 1986-02-26 04:00:27.123456789
+1987-02-21 19:48:29 1986-11-14 08:25:55.876543211 1987-06-01 08:11:02.123456789 1987-06-01 08:11:02.123456789 1986-11-14 08:25:55.876543211 1986-11-14 08:25:55.876543211 1987-06-01 08:11:02.123456789
+1987-05-28 13:52:07.900916635 1987-02-18 01:29:34.777459846 1987-09-05 01:14:41.024373424 1987-09-05 01:14:41.024373424 1987-02-18 01:29:34.777459846 1987-02-18 01:29:34.777459846 1987-09-05 01:14:41.024373424
+1998-10-16 20:05:29.397591987 1998-07-09 08:42:56.274135198 1999-01-24 06:28:02.521048776 1999-01-24 06:28:02.521048776 1998-07-09 08:42:56.274135198 1998-07-09 08:42:56.274135198 1999-01-24 06:28:02.521048776
+1999-10-03 16:59:10.396903939 1999-06-26 05:36:37.27344715 2000-01-11 03:21:43.520360728 2000-01-11 03:21:43.520360728 1999-06-26 05:36:37.27344715 1999-06-26 05:36:37.27344715 2000-01-11 03:21:43.520360728
+2000-12-18 08:42:30.000595596 2000-09-09 22:19:56.877138807 2001-03-27 20:05:03.124052385 2001-03-27 20:05:03.124052385 2000-09-09 22:19:56.877138807 2000-09-09 22:19:56.877138807 2001-03-27 20:05:03.124052385
+2002-05-10 05:29:48.990818073 2002-01-30 17:07:15.867361284 2002-08-17 16:52:22.114274862 2002-08-17 16:52:22.114274862 2002-01-30 17:07:15.867361284 2002-01-30 17:07:15.867361284 2002-08-17 16:52:22.114274862
+2003-09-23 22:33:17.00003252 2003-06-16 11:10:43.876575731 2004-01-01 08:55:50.123489309 2004-01-01 08:55:50.123489309 2003-06-16 11:10:43.876575731 2003-06-16 11:10:43.876575731 2004-01-01 08:55:50.123489309
+2004-03-07 20:14:13 2003-11-29 08:51:39.876543211 2004-06-15 08:36:46.123456789 2004-06-15 08:36:46.123456789 2003-11-29 08:51:39.876543211 2003-11-29 08:51:39.876543211 2004-06-15 08:36:46.123456789
+2007-02-09 05:17:29.368756876 2006-11-01 17:54:56.245300087 2007-05-19 17:40:02.492213665 2007-05-19 17:40:02.492213665 2006-11-01 17:54:56.245300087 2006-11-01 17:54:56.245300087 2007-05-19 17:40:02.492213665
+2009-01-21 10:49:07.108 2008-10-14 00:26:33.984543211 2009-04-30 23:11:40.231456789 2009-04-30 23:11:40.231456789 2008-10-14 00:26:33.984543211 2008-10-14 00:26:33.984543211 2009-04-30 23:11:40.231456789
+2010-04-08 02:43:35.861742727 2009-12-29 14:21:02.738285938 2010-07-16 14:06:08.985199516 2010-07-16 14:06:08.985199516 2009-12-29 14:21:02.738285938 2009-12-29 14:21:02.738285938 2010-07-16 14:06:08.985199516
+2013-04-07 02:44:43.00086821 2012-12-28 14:22:09.877411421 2013-07-15 14:07:16.124324999 2013-07-15 14:07:16.124324999 2012-12-28 14:22:09.877411421 2012-12-28 14:22:09.877411421 2013-07-15 14:07:16.124324999
+2013-04-10 00:43:46.854731546 2012-12-31 12:21:13.731274757 2013-07-18 12:06:19.978188335 2013-07-18 12:06:19.978188335 2012-12-31 12:21:13.731274757 2012-12-31 12:21:13.731274757 2013-07-18 12:06:19.978188335
+2021-09-24 03:18:32.413655165 2021-06-16 15:55:59.290198376 2022-01-01 13:41:05.537111954 2022-01-01 13:41:05.537111954 2021-06-16 15:55:59.290198376 2021-06-16 15:55:59.290198376 2022-01-01 13:41:05.537111954
+2024-11-11 16:42:41.101 2024-08-04 06:20:07.977543211 2025-02-19 04:05:14.224456789 2025-02-19 04:05:14.224456789 2024-08-04 06:20:07.977543211 2024-08-04 06:20:07.977543211 2025-02-19 04:05:14.224456789
+4143-07-08 10:53:27.252802259 4143-03-30 23:30:54.12934547 4143-10-15 22:16:00.376259048 4143-10-15 22:16:00.376259048 4143-03-30 23:30:54.12934547 4143-03-30 23:30:54.12934547 4143-10-15 22:16:00.376259048
+4966-12-04 09:30:55.202 4966-08-26 23:08:22.078543211 4967-03-13 21:53:28.325456789 4967-03-13 21:53:28.325456789 4966-08-26 23:08:22.078543211 4966-08-26 23:08:22.078543211 4967-03-13 21:53:28.325456789
+5339-02-01 14:10:01.085678691 5338-10-25 03:47:27.962221902 5339-05-12 02:32:34.20913548 5339-05-12 02:32:34.20913548 5338-10-25 03:47:27.962221902 5338-10-25 03:47:27.962221902 5339-05-12 02:32:34.20913548
+5344-10-04 18:40:08.165 5344-06-27 07:17:35.041543211 5345-01-12 05:02:41.288456789 5345-01-12 05:02:41.288456789 5344-06-27 07:17:35.041543211 5344-06-27 07:17:35.041543211 5345-01-12 05:02:41.288456789
+5397-07-13 07:12:32.000896438 5397-04-04 19:49:58.877439649 5397-10-20 18:35:05.124353227 5397-10-20 18:35:05.124353227 5397-04-04 19:49:58.877439649 5397-04-04 19:49:58.877439649 5397-10-20 18:35:05.124353227
+5966-07-09 03:30:50.597 5966-03-31 16:08:17.473543211 5966-10-16 14:53:23.720456789 5966-10-16 14:53:23.720456789 5966-03-31 16:08:17.473543211 5966-03-31 16:08:17.473543211 5966-10-16 14:53:23.720456789
+6229-06-28 02:54:28.970117179 6229-03-20 15:31:55.84666039 6229-10-05 14:17:02.093573968 6229-10-05 14:17:02.093573968 6229-03-20 15:31:55.84666039 6229-03-20 15:31:55.84666039 6229-10-05 14:17:02.093573968
+6482-04-27 12:07:38.073915413 6482-01-17 23:45:04.950458624 6482-08-04 23:30:11.197372202 6482-08-04 23:30:11.197372202 6482-01-17 23:45:04.950458624 6482-01-17 23:45:04.950458624 6482-08-04 23:30:11.197372202
+6631-11-13 16:31:29.702202248 6631-08-06 06:08:56.578745459 6632-02-21 03:54:02.825659037 6632-02-21 03:54:02.825659037 6631-08-06 06:08:56.578745459 6631-08-06 06:08:56.578745459 6632-02-21 03:54:02.825659037
+6705-09-28 18:27:28.000845672 6705-06-21 07:04:54.877388883 6706-01-06 04:50:01.124302461 6706-01-06 04:50:01.124302461 6705-06-21 07:04:54.877388883 6705-06-21 07:04:54.877388883 6706-01-06 04:50:01.124302461
+6731-02-12 08:12:48.287783702 6730-11-04 20:50:15.164326913 6731-05-22 20:35:21.411240491 6731-05-22 20:35:21.411240491 6730-11-04 20:50:15.164326913 6730-11-04 20:50:15.164326913 6731-05-22 20:35:21.411240491
+7160-12-02 06:00:24.81200852 7160-08-24 19:37:51.688551731 7161-03-11 17:22:57.935465309 7161-03-11 17:22:57.935465309 7160-08-24 19:37:51.688551731 7160-08-24 19:37:51.688551731 7161-03-11 17:22:57.935465309
+7409-09-07 23:33:32.459349602 7409-05-31 12:10:59.335892813 7409-12-16 09:56:05.582806391 7409-12-16 09:56:05.582806391 7409-05-31 12:10:59.335892813 7409-05-31 12:10:59.335892813 7409-12-16 09:56:05.582806391
+7503-06-23 23:14:17.486 7503-03-16 11:51:44.362543211 7503-10-01 10:36:50.609456789 7503-10-01 10:36:50.609456789 7503-03-16 11:51:44.362543211 7503-03-16 11:51:44.362543211 7503-10-01 10:36:50.609456789
+8422-07-22 03:21:45.745036084 8422-04-13 15:59:12.621579295 8422-10-29 14:44:18.868492873 8422-10-29 14:44:18.868492873 8422-04-13 15:59:12.621579295 8422-04-13 15:59:12.621579295 8422-10-29 14:44:18.868492873
+8521-01-16 20:42:05.668832388 8520-10-09 10:19:32.545375599 8521-04-26 09:04:38.792289177 8521-04-26 09:04:38.792289177 8520-10-09 10:19:32.545375599 8520-10-09 10:19:32.545375599 8521-04-26 09:04:38.792289177
+9075-06-13 16:20:09.218517797 9075-03-06 03:57:36.095061008 9075-09-21 03:42:42.341974586 9075-09-21 03:42:42.341974586 9075-03-06 03:57:36.095061008 9075-03-06 03:57:36.095061008 9075-09-21 03:42:42.341974586
+9209-11-11 04:08:58.223768453 9209-08-03 17:46:25.100311664 9210-02-18 15:31:31.347225242 9210-02-18 15:31:31.347225242 9209-08-03 17:46:25.100311664 9209-08-03 17:46:25.100311664 9210-02-18 15:31:31.347225242
+9403-01-09 18:12:33.547 9402-10-02 07:50:00.423543211 9403-04-19 06:35:06.670456789 9403-04-19 06:35:06.670456789 9402-10-02 07:50:00.423543211 9402-10-02 07:50:00.423543211 9403-04-19 06:35:06.670456789
+PREHOOK: query: explain
+select
+ interval '99 11:22:33.123456789' day to second + interval '10 9:8:7.123456789' day to second,
+ interval '99 11:22:33.123456789' day to second - interval '10 9:8:7.123456789' day to second
+from interval_arithmetic_1
+limit 2
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select
+ interval '99 11:22:33.123456789' day to second + interval '10 9:8:7.123456789' day to second,
+ interval '99 11:22:33.123456789' day to second - interval '10 9:8:7.123456789' day to second
+from interval_arithmetic_1
+limit 2
+POSTHOOK: type: QUERY
+Explain
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: interval_arithmetic_1
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: 109 20:30:40.246913578 (type: interval_day_time), 89 02:14:26.000000000 (type: interval_day_time)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 50 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Limit
+ Number of rows: 2
+ Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 2
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select
+ interval '99 11:22:33.123456789' day to second + interval '10 9:8:7.123456789' day to second,
+ interval '99 11:22:33.123456789' day to second - interval '10 9:8:7.123456789' day to second
+from interval_arithmetic_1
+limit 2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@interval_arithmetic_1
+#### A masked pattern was here ####
+POSTHOOK: query: select
+ interval '99 11:22:33.123456789' day to second + interval '10 9:8:7.123456789' day to second,
+ interval '99 11:22:33.123456789' day to second - interval '10 9:8:7.123456789' day to second
+from interval_arithmetic_1
+limit 2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@interval_arithmetic_1
+#### A masked pattern was here ####
+_c0 _c1
+109 20:30:40.246913578 89 02:14:26.000000000
+109 20:30:40.246913578 89 02:14:26.000000000
+PREHOOK: query: drop table interval_arithmetic_1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@interval_arithmetic_1
+PREHOOK: Output: default@interval_arithmetic_1
+POSTHOOK: query: drop table interval_arithmetic_1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@interval_arithmetic_1
+POSTHOOK: Output: default@interval_arithmetic_1
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/test/results/clientpositive/vectorized_casts.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vectorized_casts.q.out b/ql/src/test/results/clientpositive/vectorized_casts.q.out
index 6256400..804653c 100644
--- a/ql/src/test/results/clientpositive/vectorized_casts.q.out
+++ b/ql/src/test/results/clientpositive/vectorized_casts.q.out
@@ -350,18 +350,18 @@ true NULL true true true NULL true false true true 11 NULL -64615982 1803053750
true NULL true true true NULL true false true true 8 NULL 890988972 -1862301000 8 NULL 1 15 NULL NULL 8 8 8 8.0 NULL 8.90988972E8 -1.862301E9 8.0 NULL 1.0 15.892 NULL NULL 8.9098899E8 NULL 1969-12-31 16:00:00.008 NULL 1970-01-10 23:29:48.972 1969-12-10 02:41:39 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL 890988972 -1862301000 8.0 NULL TRUE 0 1969-12-31 16:00:15.892 XylAH4 XylAH4 XylAH4 8.0 1.781977944E9 0.9893582466233818 8.90988973E8
true NULL true true true NULL true false true true 8 NULL 930867246 1205399250 8 NULL 1 15 NULL NULL 8 8 8 8.0 NULL 9.30867246E8 1.20539925E9 8.0 NULL 1.0 15.892 NULL NULL 9.3086726E8 NULL 1969-12-31 16:00:00.008 NULL 1970-01-11 10:34:27.246 1970-01-14 14:49:59.25 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL 930867246 1205399250 8.0 NULL TRUE 0 1969-12-31 16:00:15.892 c1V8o1A c1V8o1A c1V8o1A 8.0 1.861734492E9 0.9893582466233818 9.30867247E8
true true NULL true true true NULL false true NULL -14 -7196 NULL -1552199500 -14 -7196 NULL 11 NULL NULL -14 -14 -14 -14.0 -7196.0 NULL -1.5521995E9 -14.0 -7196.0 NULL 11.065 NULL NULL NULL -7196.0 1969-12-31 15:59:59.986 1969-12-31 15:59:52.804 NULL 1969-12-13 16:50:00.5 1969-12-31 15:59:46 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:11.065 NULL NULL -14 -7196 NULL -1552199500 -14.0 -7196.0 NULL 0 1969-12-31 16:00:11.065 NULL NULL NULL -14.0 NULL -0.9906073556948704 NULL
-true true NULL true true true NULL false true NULL -21 -7196 NULL 1542429000 -21 -7196 NULL -4 NULL NULL -21 -21 -21 -21.0 -7196.0 NULL 1.542429E9 -21.0 -7196.0 NULL -4.1 NULL NULL NULL -7196.0 1969-12-31 15:59:59.979 1969-12-31 15:59:52.804 NULL 1970-01-18 12:27:09 1969-12-31 15:59:39 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:55.9 NULL NULL -21 -7196 NULL 1542429000 -21.0 -7196.0 NULL 0 1969-12-31 15:59:55.9 NULL NULL NULL -21.0 NULL -0.8366556385360561 NULL
-true true NULL true true true NULL false true NULL -24 -7196 NULL 829111000 -24 -7196 NULL -6 NULL NULL -24 -24 -24 -24.0 -7196.0 NULL 8.29111E8 -24.0 -7196.0 NULL -6.855 NULL NULL NULL -7196.0 1969-12-31 15:59:59.976 1969-12-31 15:59:52.804 NULL 1970-01-10 06:18:31 1969-12-31 15:59:36 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:53.145 NULL NULL -24 -7196 NULL 829111000 -24.0 -7196.0 NULL 0 1969-12-31 15:59:53.145 NULL NULL NULL -24.0 NULL 0.9055783620066238 NULL
+true true NULL true true true NULL false true NULL -21 -7196 NULL 1542429000 -21 -7196 NULL -5 NULL NULL -21 -21 -21 -21.0 -7196.0 NULL 1.542429E9 -21.0 -7196.0 NULL -4.1 NULL NULL NULL -7196.0 1969-12-31 15:59:59.979 1969-12-31 15:59:52.804 NULL 1970-01-18 12:27:09 1969-12-31 15:59:39 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:55.9 NULL NULL -21 -7196 NULL 1542429000 -21.0 -7196.0 NULL 0 1969-12-31 15:59:55.9 NULL NULL NULL -21.0 NULL -0.8366556385360561 NULL
+true true NULL true true true NULL false true NULL -24 -7196 NULL 829111000 -24 -7196 NULL -7 NULL NULL -24 -24 -24 -24.0 -7196.0 NULL 8.29111E8 -24.0 -7196.0 NULL -6.855 NULL NULL NULL -7196.0 1969-12-31 15:59:59.976 1969-12-31 15:59:52.804 NULL 1970-01-10 06:18:31 1969-12-31 15:59:36 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:53.145 NULL NULL -24 -7196 NULL 829111000 -24.0 -7196.0 NULL 0 1969-12-31 15:59:53.145 NULL NULL NULL -24.0 NULL 0.9055783620066238 NULL
true true NULL true true true NULL false true NULL -30 -200 NULL 1429852250 -30 -200 NULL 12 NULL NULL -30 -30 -30 -30.0 -200.0 NULL 1.42985225E9 -30.0 -200.0 NULL 12.935 NULL NULL NULL -200.0 1969-12-31 15:59:59.97 1969-12-31 15:59:59.8 NULL 1970-01-17 05:10:52.25 1969-12-31 15:59:30 1969-12-31 15:56:40 NULL 1969-12-31 16:00:00 1969-12-31 16:00:12.935 NULL NULL -30 -200 NULL 1429852250 -30.0 -200.0 NULL 0 1969-12-31 16:00:12.935 NULL NULL NULL -30.0 NULL 0.9880316240928618 NULL
-true true NULL true true true NULL false true NULL -36 -200 NULL -2006216750 -36 -200 NULL -14 NULL NULL -36 -36 -36 -36.0 -200.0 NULL -2.00621675E9 -36.0 -200.0 NULL -14.252 NULL NULL NULL -200.0 1969-12-31 15:59:59.964 1969-12-31 15:59:59.8 NULL 1969-12-08 10:43:03.25 1969-12-31 15:59:24 1969-12-31 15:56:40 NULL 1969-12-31 16:00:00 1969-12-31 15:59:45.748 NULL NULL -36 -200 NULL -2006216750 -36.0 -200.0 NULL 0 1969-12-31 15:59:45.748 NULL NULL NULL -36.0 NULL 0.9917788534431158 NULL
-true true NULL true true true NULL false true NULL -36 -200 NULL 1599879000 -36 -200 NULL -6 NULL NULL -36 -36 -36 -36.0 -200.0 NULL 1.599879E9 -36.0 -200.0 NULL -6.183 NULL NULL NULL -200.0 1969-12-31 15:59:59.964 1969-12-31 15:59:59.8 NULL 1970-01-19 04:24:39 1969-12-31 15:59:24 1969-12-31 15:56:40 NULL 1969-12-31 16:00:00 1969-12-31 15:59:53.817 NULL NULL -36 -200 NULL 1599879000 -36.0 -200.0 NULL 0 1969-12-31 15:59:53.817 NULL NULL NULL -36.0 NULL 0.9917788534431158 NULL
-true true NULL true true true NULL false true NULL -38 15601 NULL -1858689000 -38 15601 NULL -1 NULL NULL -38 -38 -38 -38.0 15601.0 NULL -1.858689E9 -38.0 15601.0 NULL -1.386 NULL NULL NULL 15601.0 1969-12-31 15:59:59.962 1969-12-31 16:00:15.601 NULL 1969-12-10 03:41:51 1969-12-31 15:59:22 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 15:59:58.614 NULL NULL -38 15601 NULL -1858689000 -38.0 15601.0 NULL 0 1969-12-31 15:59:58.614 NULL NULL NULL -38.0 NULL -0.2963685787093853 NULL
+true true NULL true true true NULL false true NULL -36 -200 NULL -2006216750 -36 -200 NULL -15 NULL NULL -36 -36 -36 -36.0 -200.0 NULL -2.00621675E9 -36.0 -200.0 NULL -14.252 NULL NULL NULL -200.0 1969-12-31 15:59:59.964 1969-12-31 15:59:59.8 NULL 1969-12-08 10:43:03.25 1969-12-31 15:59:24 1969-12-31 15:56:40 NULL 1969-12-31 16:00:00 1969-12-31 15:59:45.748 NULL NULL -36 -200 NULL -2006216750 -36.0 -200.0 NULL 0 1969-12-31 15:59:45.748 NULL NULL NULL -36.0 NULL 0.9917788534431158 NULL
+true true NULL true true true NULL false true NULL -36 -200 NULL 1599879000 -36 -200 NULL -7 NULL NULL -36 -36 -36 -36.0 -200.0 NULL 1.599879E9 -36.0 -200.0 NULL -6.183 NULL NULL NULL -200.0 1969-12-31 15:59:59.964 1969-12-31 15:59:59.8 NULL 1970-01-19 04:24:39 1969-12-31 15:59:24 1969-12-31 15:56:40 NULL 1969-12-31 16:00:00 1969-12-31 15:59:53.817 NULL NULL -36 -200 NULL 1599879000 -36.0 -200.0 NULL 0 1969-12-31 15:59:53.817 NULL NULL NULL -36.0 NULL 0.9917788534431158 NULL
+true true NULL true true true NULL false true NULL -38 15601 NULL -1858689000 -38 15601 NULL -2 NULL NULL -38 -38 -38 -38.0 15601.0 NULL -1.858689E9 -38.0 15601.0 NULL -1.3860000000000001 NULL NULL NULL 15601.0 1969-12-31 15:59:59.962 1969-12-31 16:00:15.601 NULL 1969-12-10 03:41:51 1969-12-31 15:59:22 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 15:59:58.614 NULL NULL -38 15601 NULL -1858689000 -38.0 15601.0 NULL 0 1969-12-31 15:59:58.614 NULL NULL NULL -38.0 NULL -0.2963685787093853 NULL
true true NULL true true true NULL false true NULL -5 15601 NULL 612416000 -5 15601 NULL 4 NULL NULL -5 -5 -5 -5.0 15601.0 NULL 6.12416E8 -5.0 15601.0 NULL 4.679 NULL NULL NULL 15601.0 1969-12-31 15:59:59.995 1969-12-31 16:00:15.601 NULL 1970-01-07 18:06:56 1969-12-31 15:59:55 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 16:00:04.679 NULL NULL -5 15601 NULL 612416000 -5.0 15601.0 NULL 0 1969-12-31 16:00:04.679 NULL NULL NULL -5.0 NULL 0.9589242746631385 NULL
-true true NULL true true true NULL false true NULL -50 -7196 NULL -1031187250 -50 -7196 NULL -5 NULL NULL -50 -50 -50 -50.0 -7196.0 NULL -1.03118725E9 -50.0 -7196.0 NULL -5.267 NULL NULL NULL -7196.0 1969-12-31 15:59:59.95 1969-12-31 15:59:52.804 NULL 1969-12-19 17:33:32.75 1969-12-31 15:59:10 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:54.733 NULL NULL -50 -7196 NULL -1031187250 -50.0 -7196.0 NULL 0 1969-12-31 15:59:54.733 NULL NULL NULL -50.0 NULL 0.26237485370392877 NULL
+true true NULL true true true NULL false true NULL -50 -7196 NULL -1031187250 -50 -7196 NULL -6 NULL NULL -50 -50 -50 -50.0 -7196.0 NULL -1.03118725E9 -50.0 -7196.0 NULL -5.267 NULL NULL NULL -7196.0 1969-12-31 15:59:59.95 1969-12-31 15:59:52.804 NULL 1969-12-19 17:33:32.75 1969-12-31 15:59:10 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:54.733 NULL NULL -50 -7196 NULL -1031187250 -50.0 -7196.0 NULL 0 1969-12-31 15:59:54.733 NULL NULL NULL -50.0 NULL 0.26237485370392877 NULL
true true NULL true true true NULL false true NULL -59 -7196 NULL -1604890000 -59 -7196 NULL 13 NULL NULL -59 -59 -59 -59.0 -7196.0 NULL -1.60489E9 -59.0 -7196.0 NULL 13.15 NULL NULL NULL -7196.0 1969-12-31 15:59:59.941 1969-12-31 15:59:52.804 NULL 1969-12-13 02:11:50 1969-12-31 15:59:01 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:13.15 NULL NULL -59 -7196 NULL -1604890000 -59.0 -7196.0 NULL 0 1969-12-31 16:00:13.15 NULL NULL NULL -59.0 NULL -0.6367380071391379 NULL
-true true NULL true true true NULL false true NULL -60 -7196 NULL 1516314750 -60 -7196 NULL -7 NULL NULL -60 -60 -60 -60.0 -7196.0 NULL 1.51631475E9 -60.0 -7196.0 NULL -7.592 NULL NULL NULL -7196.0 1969-12-31 15:59:59.94 1969-12-31 15:59:52.804 NULL 1970-01-18 05:11:54.75 1969-12-31 15:59:00 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:52.408 NULL NULL -60 -7196 NULL 1516314750 -60.0 -7196.0 NULL 0 1969-12-31 15:59:52.408 NULL NULL NULL -60.0 NULL 0.3048106211022167 NULL
+true true NULL true true true NULL false true NULL -60 -7196 NULL 1516314750 -60 -7196 NULL -8 NULL NULL -60 -60 -60 -60.0 -7196.0 NULL 1.51631475E9 -60.0 -7196.0 NULL -7.592 NULL NULL NULL -7196.0 1969-12-31 15:59:59.94 1969-12-31 15:59:52.804 NULL 1970-01-18 05:11:54.75 1969-12-31 15:59:00 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:52.408 NULL NULL -60 -7196 NULL 1516314750 -60.0 -7196.0 NULL 0 1969-12-31 15:59:52.408 NULL NULL NULL -60.0 NULL 0.3048106211022167 NULL
true true NULL true true true NULL false true NULL -8 -7196 NULL -1849991500 -8 -7196 NULL 3 NULL NULL -8 -8 -8 -8.0 -7196.0 NULL -1.8499915E9 -8.0 -7196.0 NULL 3.136 NULL NULL NULL -7196.0 1969-12-31 15:59:59.992 1969-12-31 15:59:52.804 NULL 1969-12-10 06:06:48.5 1969-12-31 15:59:52 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:03.136 NULL NULL -8 -7196 NULL -1849991500 -8.0 -7196.0 NULL 0 1969-12-31 16:00:03.136 NULL NULL NULL -8.0 NULL -0.9893582466233818 NULL
-true true NULL true true true NULL false true NULL 20 15601 NULL -362433250 20 15601 NULL -14 NULL NULL 20 20 20 20.0 15601.0 NULL -3.6243325E8 20.0 15601.0 NULL -14.871 NULL NULL NULL 15601.0 1969-12-31 16:00:00.02 1969-12-31 16:00:15.601 NULL 1969-12-27 11:19:26.75 1969-12-31 16:00:20 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 15:59:45.129 NULL NULL 20 15601 NULL -362433250 20.0 15601.0 NULL 0 1969-12-31 15:59:45.129 NULL NULL NULL 20.0 NULL 0.9129452507276277 NULL
-true true NULL true true true NULL false true NULL 48 15601 NULL -795361000 48 15601 NULL -9 NULL NULL 48 48 48 48.0 15601.0 NULL -7.95361E8 48.0 15601.0 NULL -9.765 NULL NULL NULL 15601.0 1969-12-31 16:00:00.048 1969-12-31 16:00:15.601 NULL 1969-12-22 11:03:59 1969-12-31 16:00:48 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 15:59:50.235 NULL NULL 48 15601 NULL -795361000 48.0 15601.0 NULL 0 1969-12-31 15:59:50.235 NULL NULL NULL 48.0 NULL -0.7682546613236668 NULL
+true true NULL true true true NULL false true NULL 20 15601 NULL -362433250 20 15601 NULL -15 NULL NULL 20 20 20 20.0 15601.0 NULL -3.6243325E8 20.0 15601.0 NULL -14.871 NULL NULL NULL 15601.0 1969-12-31 16:00:00.02 1969-12-31 16:00:15.601 NULL 1969-12-27 11:19:26.75 1969-12-31 16:00:20 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 15:59:45.129 NULL NULL 20 15601 NULL -362433250 20.0 15601.0 NULL 0 1969-12-31 15:59:45.129 NULL NULL NULL 20.0 NULL 0.9129452507276277 NULL
+true true NULL true true true NULL false true NULL 48 15601 NULL -795361000 48 15601 NULL -10 NULL NULL 48 48 48 48.0 15601.0 NULL -7.95361E8 48.0 15601.0 NULL -9.765 NULL NULL NULL 15601.0 1969-12-31 16:00:00.048 1969-12-31 16:00:15.601 NULL 1969-12-22 11:03:59 1969-12-31 16:00:48 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 15:59:50.235 NULL NULL 48 15601 NULL -795361000 48.0 15601.0 NULL 0 1969-12-31 15:59:50.235 NULL NULL NULL 48.0 NULL -0.7682546613236668 NULL
true true NULL true true true NULL false true NULL 5 -7196 NULL -1015607500 5 -7196 NULL 10 NULL NULL 5 5 5 5.0 -7196.0 NULL -1.0156075E9 5.0 -7196.0 NULL 10.973 NULL NULL NULL -7196.0 1969-12-31 16:00:00.005 1969-12-31 15:59:52.804 NULL 1969-12-19 21:53:12.5 1969-12-31 16:00:05 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:10.973 NULL NULL 5 -7196 NULL -1015607500 5.0 -7196.0 NULL 0 1969-12-31 16:00:10.973 NULL NULL NULL 5.0 NULL -0.9589242746631385 NULL
true true NULL true true true NULL false true NULL 59 -7196 NULL -1137754500 59 -7196 NULL 10 NULL NULL 59 59 59 59.0 -7196.0 NULL -1.1377545E9 59.0 -7196.0 NULL 10.956 NULL NULL NULL -7196.0 1969-12-31 16:00:00.059 1969-12-31 15:59:52.804 NULL 1969-12-18 11:57:25.5 1969-12-31 16:00:59 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:10.956 NULL NULL 59 -7196 NULL -1137754500 59.0 -7196.0 NULL 0 1969-12-31 16:00:10.956 NULL NULL NULL 59.0 NULL 0.6367380071391379 NULL
[17/50] [abbrv] hive git commit: HIVE-13324. LLAP: history log for
FRAGMENT_START doesn't log DagId correctly. (Siddharth Seth,
Reviewed by Sergey Shelukhin)
Posted by jd...@apache.org.
HIVE-13324. LLAP: history log for FRAGMENT_START doesn't log DagId correctly. (Siddharth Seth, Reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3038b05e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3038b05e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3038b05e
Branch: refs/heads/llap
Commit: 3038b05ed346f4b5438e9072edb19186ea90d042
Parents: 2449d1d
Author: Siddharth Seth <ss...@apache.org>
Authored: Sat Mar 26 14:12:36 2016 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Sat Mar 26 14:12:36 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/llap/tez/Converters.java | 1 +
.../hadoop/hive/llap/tez/TestConverters.java | 190 +++++++++++++++++++
2 files changed, 191 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/3038b05e/llap-common/src/java/org/apache/hadoop/hive/llap/tez/Converters.java
----------------------------------------------------------------------
diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/tez/Converters.java b/llap-common/src/java/org/apache/hadoop/hive/llap/tez/Converters.java
index a5c3631..ec6e439 100644
--- a/llap-common/src/java/org/apache/hadoop/hive/llap/tez/Converters.java
+++ b/llap-common/src/java/org/apache/hadoop/hive/llap/tez/Converters.java
@@ -85,6 +85,7 @@ public class Converters {
FragmentSpecProto.Builder builder = FragmentSpecProto.newBuilder();
builder.setFragmentIdentifierString(taskSpec.getTaskAttemptID().toString());
builder.setDagName(taskSpec.getDAGName());
+ builder.setDagId(taskSpec.getDagIdentifier());
builder.setVertexName(taskSpec.getVertexName());
builder.setVertexParallelism(taskSpec.getVertexParallelism());
builder.setFragmentNumber(taskSpec.getTaskAttemptID().getTaskID().getId());
http://git-wip-us.apache.org/repos/asf/hive/blob/3038b05e/llap-common/src/test/org/apache/hadoop/hive/llap/tez/TestConverters.java
----------------------------------------------------------------------
diff --git a/llap-common/src/test/org/apache/hadoop/hive/llap/tez/TestConverters.java b/llap-common/src/test/org/apache/hadoop/hive/llap/tez/TestConverters.java
new file mode 100644
index 0000000..d4cdac1
--- /dev/null
+++ b/llap-common/src/test/org/apache/hadoop/hive/llap/tez/TestConverters.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.tez;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+
+import com.google.common.collect.Lists;
+import com.google.protobuf.ByteString;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.FragmentSpecProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.dag.api.InputDescriptor;
+import org.apache.tez.dag.api.OutputDescriptor;
+import org.apache.tez.dag.api.ProcessorDescriptor;
+import org.apache.tez.dag.api.UserPayload;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.records.TezVertexID;
+import org.apache.tez.runtime.api.impl.InputSpec;
+import org.apache.tez.runtime.api.impl.OutputSpec;
+import org.apache.tez.runtime.api.impl.TaskSpec;
+import org.junit.Test;
+
+public class TestConverters {
+
+ @Test(timeout = 5000)
+ public void testTaskSpecToFragmentSpec() {
+ ByteBuffer procBb = ByteBuffer.allocate(4);
+ procBb.putInt(0, 200);
+ UserPayload processorPayload = UserPayload.create(procBb);
+ ProcessorDescriptor processorDescriptor =
+ ProcessorDescriptor.create("fakeProcessorName").setUserPayload(processorPayload);
+
+ ByteBuffer input1Bb = ByteBuffer.allocate(4);
+ input1Bb.putInt(0, 300);
+ UserPayload input1Payload = UserPayload.create(input1Bb);
+ InputDescriptor id1 = InputDescriptor.create("input1ClassName").setUserPayload(input1Payload);
+ InputSpec inputSpec1 = new InputSpec("sourceVertexName1", id1, 33);
+ InputSpec inputSpec2 = new InputSpec("sourceVertexName2", id1, 44);
+ List<InputSpec> inputSpecList = Lists.newArrayList(inputSpec1, inputSpec2);
+
+ ByteBuffer output1Bb = ByteBuffer.allocate(4);
+ output1Bb.putInt(0, 400);
+ UserPayload output1Payload = UserPayload.create(output1Bb);
+ OutputDescriptor od1 =
+ OutputDescriptor.create("output1ClassName").setUserPayload(output1Payload);
+ OutputSpec outputSpec1 = new OutputSpec("destVertexName1", od1, 55);
+ OutputSpec outputSpec2 = new OutputSpec("destVertexName2", od1, 66);
+ List<OutputSpec> outputSpecList = Lists.newArrayList(outputSpec1, outputSpec2);
+
+ ApplicationId appId = ApplicationId.newInstance(1000, 100);
+ TezDAGID tezDagId = TezDAGID.getInstance(appId, 300);
+ TezVertexID tezVertexId = TezVertexID.getInstance(tezDagId, 400);
+ TezTaskID tezTaskId = TezTaskID.getInstance(tezVertexId, 500);
+ TezTaskAttemptID tezTaskAttemptId = TezTaskAttemptID.getInstance(tezTaskId, 600);
+
+ TaskSpec taskSpec =
+ new TaskSpec(tezTaskAttemptId, "dagName", "vertexName", 10, processorDescriptor,
+ inputSpecList, outputSpecList, null);
+
+
+ FragmentSpecProto fragmentSpecProto = Converters.convertTaskSpecToProto(taskSpec);
+
+
+ assertEquals("dagName", fragmentSpecProto.getDagName());
+ assertEquals("vertexName", fragmentSpecProto.getVertexName());
+ assertEquals(tezTaskAttemptId.toString(), fragmentSpecProto.getFragmentIdentifierString());
+ assertEquals(tezDagId.getId(), fragmentSpecProto.getDagId());
+ assertEquals(tezTaskAttemptId.getId(), fragmentSpecProto.getAttemptNumber());
+ assertEquals(tezTaskId.getId(), fragmentSpecProto.getFragmentNumber());
+ assertEquals(processorDescriptor.getClassName(),
+ fragmentSpecProto.getProcessorDescriptor().getClassName());
+ assertEquals(processorDescriptor.getUserPayload().getPayload(),
+ fragmentSpecProto.getProcessorDescriptor().getUserPayload().getUserPayload()
+ .asReadOnlyByteBuffer());
+ assertEquals(2, fragmentSpecProto.getInputSpecsCount());
+ assertEquals(2, fragmentSpecProto.getOutputSpecsCount());
+
+ verifyInputSpecAndProto(inputSpec1, fragmentSpecProto.getInputSpecs(0));
+ verifyInputSpecAndProto(inputSpec2, fragmentSpecProto.getInputSpecs(1));
+ verifyOutputSpecAndProto(outputSpec1, fragmentSpecProto.getOutputSpecs(0));
+ verifyOutputSpecAndProto(outputSpec2, fragmentSpecProto.getOutputSpecs(1));
+
+ }
+
+ @Test (timeout = 5000)
+ public void testFragmentSpecToTaskSpec() {
+
+ ByteBuffer procBb = ByteBuffer.allocate(4);
+ procBb.putInt(0, 200);
+
+ ByteBuffer input1Bb = ByteBuffer.allocate(4);
+ input1Bb.putInt(0, 300);
+
+ ByteBuffer output1Bb = ByteBuffer.allocate(4);
+ output1Bb.putInt(0, 400);
+
+ ApplicationId appId = ApplicationId.newInstance(1000, 100);
+ TezDAGID tezDagId = TezDAGID.getInstance(appId, 300);
+ TezVertexID tezVertexId = TezVertexID.getInstance(tezDagId, 400);
+ TezTaskID tezTaskId = TezTaskID.getInstance(tezVertexId, 500);
+ TezTaskAttemptID tezTaskAttemptId = TezTaskAttemptID.getInstance(tezTaskId, 600);
+
+ FragmentSpecProto.Builder builder = FragmentSpecProto.newBuilder();
+ builder.setFragmentIdentifierString(tezTaskAttemptId.toString());
+ builder.setDagName("dagName");
+ builder.setVertexName("vertexName");
+ builder.setDagId(tezDagId.getId());
+ builder.setProcessorDescriptor(
+ EntityDescriptorProto.newBuilder().setClassName("fakeProcessorName").setUserPayload(
+ UserPayloadProto.newBuilder().setUserPayload(ByteString.copyFrom(procBb))));
+ builder.addInputSpecs(IOSpecProto.newBuilder().setConnectedVertexName("sourceVertexName1")
+ .setPhysicalEdgeCount(33).setIoDescriptor(
+ EntityDescriptorProto.newBuilder().setClassName("input1ClassName").setUserPayload(
+ UserPayloadProto.newBuilder().setUserPayload(ByteString.copyFrom(input1Bb)))));
+ builder.addInputSpecs(IOSpecProto.newBuilder().setConnectedVertexName("sourceVertexName2")
+ .setPhysicalEdgeCount(44).setIoDescriptor(
+ EntityDescriptorProto.newBuilder().setClassName("input1ClassName").setUserPayload(
+ UserPayloadProto.newBuilder().setUserPayload(ByteString.copyFrom(input1Bb)))));
+ builder.addOutputSpecs(IOSpecProto.newBuilder().setConnectedVertexName("destVertexName1")
+ .setPhysicalEdgeCount(55).setIoDescriptor(
+ EntityDescriptorProto.newBuilder().setClassName("outputClassName").setUserPayload(
+ UserPayloadProto.newBuilder().setUserPayload(ByteString.copyFrom(output1Bb)))));
+ builder.addOutputSpecs(IOSpecProto.newBuilder().setConnectedVertexName("destVertexName2")
+ .setPhysicalEdgeCount(66).setIoDescriptor(
+ EntityDescriptorProto.newBuilder().setClassName("outputClassName").setUserPayload(
+ UserPayloadProto.newBuilder().setUserPayload(ByteString.copyFrom(output1Bb)))));
+
+ FragmentSpecProto fragmentSpecProto = builder.build();
+
+ TaskSpec taskSpec = Converters.getTaskSpecfromProto(fragmentSpecProto);
+
+ assertEquals("dagName", taskSpec.getDAGName());
+ assertEquals("vertexName", taskSpec.getVertexName());
+ assertEquals(tezTaskAttemptId, taskSpec.getTaskAttemptID());
+ assertEquals("fakeProcessorName", taskSpec.getProcessorDescriptor().getClassName());
+ byte[] serialized = new byte[taskSpec.getProcessorDescriptor().getUserPayload().getPayload().remaining()];
+ taskSpec.getProcessorDescriptor().getUserPayload().getPayload().get(serialized);
+ assertArrayEquals(procBb.array(), serialized);
+
+ assertEquals(2, taskSpec.getInputs().size());
+ assertEquals(2, taskSpec.getOutputs().size());
+
+ verifyInputSpecAndProto(taskSpec.getInputs().get(0), fragmentSpecProto.getInputSpecs(0));
+ verifyInputSpecAndProto(taskSpec.getInputs().get(1), fragmentSpecProto.getInputSpecs(1));
+ verifyOutputSpecAndProto(taskSpec.getOutputs().get(0), fragmentSpecProto.getOutputSpecs(0));
+ verifyOutputSpecAndProto(taskSpec.getOutputs().get(1), fragmentSpecProto.getOutputSpecs(1));
+
+
+ }
+
+ private void verifyInputSpecAndProto(InputSpec inputSpec,
+ IOSpecProto inputSpecProto) {
+ assertEquals(inputSpec.getPhysicalEdgeCount(), inputSpecProto.getPhysicalEdgeCount());
+ assertEquals(inputSpec.getSourceVertexName(), inputSpecProto.getConnectedVertexName());
+ assertEquals(inputSpec.getInputDescriptor().getClassName(),
+ inputSpecProto.getIoDescriptor().getClassName());
+ assertEquals(inputSpec.getInputDescriptor().getUserPayload().getPayload(),
+ inputSpecProto.getIoDescriptor().getUserPayload().getUserPayload().asReadOnlyByteBuffer());
+ }
+
+ private void verifyOutputSpecAndProto(OutputSpec outputSpec,
+ IOSpecProto outputSpecProto) {
+ assertEquals(outputSpec.getPhysicalEdgeCount(), outputSpecProto.getPhysicalEdgeCount());
+ assertEquals(outputSpec.getDestinationVertexName(), outputSpecProto.getConnectedVertexName());
+ assertEquals(outputSpec.getOutputDescriptor().getClassName(),
+ outputSpecProto.getIoDescriptor().getClassName());
+ assertEquals(outputSpec.getOutputDescriptor().getUserPayload().getPayload(),
+ outputSpecProto.getIoDescriptor().getUserPayload().getUserPayload().asReadOnlyByteBuffer());
+ }
+}
[49/50] [abbrv] hive git commit: HIVE-10249 ACID: show locks should
show who the lock is waiting for (Eugene Koifman, reviewed by Wei Zheng)
Posted by jd...@apache.org.
HIVE-10249 ACID: show locks should show who the lock is waiting for (Eugene Koifman, reviewed by Wei Zheng)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4e9f95a1
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4e9f95a1
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4e9f95a1
Branch: refs/heads/llap
Commit: 4e9f95a1bad89ac4ea0cefc65eeba7a1e56a948d
Parents: 51efcb8
Author: Eugene Koifman <ek...@hortonworks.com>
Authored: Wed Mar 30 12:17:06 2016 -0700
Committer: Eugene Koifman <ek...@hortonworks.com>
Committed: Wed Mar 30 12:17:06 2016 -0700
----------------------------------------------------------------------
.../hadoop/hive/metastore/txn/TxnDbUtil.java | 6 ++-
.../hadoop/hive/metastore/txn/TxnHandler.java | 46 ++++++++++++++++----
.../hive/metastore/txn/TestTxnHandler.java | 2 +-
.../org/apache/hadoop/hive/ql/exec/DDLTask.java | 16 ++++++-
.../hive/ql/lockmgr/TestDbTxnManager2.java | 28 ++++++++++++
.../clientpositive/dbtxnmgr_showlocks.q.out | 6 +--
6 files changed, 89 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/4e9f95a1/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java
index df480ea..c82d23a 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java
@@ -103,7 +103,11 @@ public final class TxnDbUtil {
" HL_ACQUIRED_AT bigint," +
" HL_USER varchar(128) NOT NULL," +
" HL_HOST varchar(128) NOT NULL," +
- " PRIMARY KEY(HL_LOCK_EXT_ID, HL_LOCK_INT_ID))");
+ " HL_HEARTBEAT_COUNT integer," +
+ " HL_AGENT_INFO varchar(128)," +
+ " HL_BLOCKEDBY_EXT_ID bigint," +
+ " HL_BLOCKEDBY_INT_ID bigint," +
+ " PRIMARY KEY(HL_LOCK_EXT_ID, HL_LOCK_INT_ID))");
stmt.execute("CREATE INDEX HL_TXNID_INDEX ON HIVE_LOCKS (HL_TXNID)");
stmt.execute("CREATE TABLE NEXT_LOCK_ID (" + " NL_NEXT bigint NOT NULL)");
http://git-wip-us.apache.org/repos/asf/hive/blob/4e9f95a1/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
index 21faff4..be3c6de 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
@@ -847,8 +847,8 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
*/
private static class LockInfoExt extends LockInfo {
private final ShowLocksResponseElement e;
- LockInfoExt(ShowLocksResponseElement e, long intLockId) {
- super(e, intLockId);
+ LockInfoExt(ShowLocksResponseElement e) {
+ super(e);
this.e = e;
}
}
@@ -864,7 +864,8 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
stmt = dbConn.createStatement();
String s = "select hl_lock_ext_id, hl_txnid, hl_db, hl_table, hl_partition, hl_lock_state, " +
- "hl_lock_type, hl_last_heartbeat, hl_acquired_at, hl_user, hl_host, hl_lock_int_id from HIVE_LOCKS";
+ "hl_lock_type, hl_last_heartbeat, hl_acquired_at, hl_user, hl_host, hl_lock_int_id," +
+ "hl_blockedby_ext_id, hl_blockedby_int_id from HIVE_LOCKS";
LOG.debug("Doing to execute query <" + s + ">");
ResultSet rs = stmt.executeQuery(s);
while (rs.next()) {
@@ -892,7 +893,16 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
if (!rs.wasNull()) e.setAcquiredat(acquiredAt);
e.setUser(rs.getString(10));
e.setHostname(rs.getString(11));
- sortedList.add(new LockInfoExt(e, rs.getLong(12)));
+ e.setLockIdInternal(rs.getLong(12));
+ long id = rs.getLong(13);
+ if(!rs.wasNull()) {
+ e.setBlockedByExtId(id);
+ }
+ id = rs.getLong(14);
+ if(!rs.wasNull()) {
+ e.setBlockedByIntId(id);
+ }
+ sortedList.add(new LockInfoExt(e));
}
LOG.debug("Going to rollback");
dbConn.rollback();
@@ -1142,6 +1152,10 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
private static void shouldNeverHappen(long txnid) {
throw new RuntimeException("This should never happen: " + JavaUtils.txnIdToString(txnid));
}
+ private static void shouldNeverHappen(long txnid, long extLockId, long intLockId) {
+ throw new RuntimeException("This should never happen: " + JavaUtils.txnIdToString(txnid) + " "
+ + JavaUtils.lockIdToString(extLockId) + " " + intLockId);
+ }
public void addDynamicPartitions(AddDynamicPartitions rqst)
throws NoSuchTxnException, TxnAbortedException, MetaException {
Connection dbConn = null;
@@ -1673,15 +1687,15 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
}
txnId = rs.getLong("hl_txnid");//returns 0 if value is NULL
}
- LockInfo(ShowLocksResponseElement e, long intLockId) {
+ LockInfo(ShowLocksResponseElement e) {
extLockId = e.getLockid();
- this.intLockId = intLockId;
+ intLockId = e.getLockIdInternal();
+ txnId = e.getTxnid();
db = e.getDbname();
table = e.getTablename();
partition = e.getPartname();
state = e.getState();
type = e.getType();
- txnId = e.getTxnid();
}
public boolean equals(Object other) {
@@ -1980,9 +1994,10 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
LOG.debug("Going to execute query <" + query.toString() + ">");
Statement stmt = null;
+ ResultSet rs = null;
try {
stmt = dbConn.createStatement();
- ResultSet rs = stmt.executeQuery(query.toString());
+ rs = stmt.executeQuery(query.toString());
SortedSet<LockInfo> lockSet = new TreeSet<LockInfo>(new LockInfoComparator());
while (rs.next()) {
lockSet.add(new LockInfo(rs));
@@ -2053,7 +2068,20 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
switch (lockAction) {
case WAIT:
if(!ignoreConflict(info, locks[i])) {
+ /*we acquire all locks for a given query atomically; if 1 blocks, all go into (remain) in
+ * Waiting state. wait() will undo any 'acquire()' which may have happened as part of
+ * this (metastore db) transaction and then we record which lock blocked the lock
+ * we were testing ('info').*/
wait(dbConn, save);
+ String sqlText = "update HIVE_LOCKS" +
+ " set HL_BLOCKEDBY_EXT_ID=" + locks[i].extLockId +
+ ", HL_BLOCKEDBY_INT_ID=" + locks[i].intLockId +
+ " where HL_LOCK_EXT_ID=" + info.extLockId + " and HL_LOCK_INT_ID=" + info.intLockId;
+ LOG.debug("Executing sql: " + sqlText);
+ int updCnt = stmt.executeUpdate(sqlText);
+ if(updCnt != 1) {
+ shouldNeverHappen(info.txnId, info.extLockId, info.intLockId);
+ }
LOG.debug("Going to commit");
dbConn.commit();
response.setState(LockState.WAITING);
@@ -2082,7 +2110,7 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
dbConn.commit();
response.setState(LockState.ACQUIRED);
} finally {
- closeStmt(stmt);
+ close(rs, stmt, null);
}
return response;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/4e9f95a1/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java b/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
index 26a660a..37eacde 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
@@ -1153,7 +1153,7 @@ public class TestTxnHandler {
}
@Test
- @Ignore
+ @Ignore("Wedges Derby")
public void deadlockDetected() throws Exception {
LOG.debug("Starting deadlock test");
if (txnHandler instanceof TxnHandler) {
http://git-wip-us.apache.org/repos/asf/hive/blob/4e9f95a1/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index 56eecf6..b26f09d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -2541,6 +2541,8 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
os.write(separator);
os.writeBytes("State");
os.write(separator);
+ os.writeBytes("Blocked By");
+ os.write(separator);
os.writeBytes("Type");
os.write(separator);
os.writeBytes("Transaction ID");
@@ -2557,7 +2559,12 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
List<ShowLocksResponseElement> locks = rsp.getLocks();
if (locks != null) {
for (ShowLocksResponseElement lock : locks) {
- os.writeBytes(Long.toString(lock.getLockid()));
+ if(lock.isSetLockIdInternal()) {
+ os.writeBytes(Long.toString(lock.getLockid()) + "." + Long.toString(lock.getLockIdInternal()));
+ }
+ else {
+ os.writeBytes(Long.toString(lock.getLockid()));
+ }
os.write(separator);
os.writeBytes(lock.getDbname());
os.write(separator);
@@ -2567,6 +2574,13 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
os.write(separator);
os.writeBytes(lock.getState().toString());
os.write(separator);
+ if(lock.isSetBlockedByExtId()) {//both "blockedby" are either there or not
+ os.writeBytes(Long.toString(lock.getBlockedByExtId()) + "." + Long.toString(lock.getBlockedByIntId()));
+ }
+ else {
+ os.writeBytes(" ");//12 chars - try to keep cols aligned
+ }
+ os.write(separator);
os.writeBytes(lock.getType().toString());
os.write(separator);
os.writeBytes((lock.getTxnid() == 0) ? "NULL" : Long.toString(lock.getTxnid()));
http://git-wip-us.apache.org/repos/asf/hive/blob/4e9f95a1/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
index d1b370e..836b507 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
@@ -240,6 +240,34 @@ public class TestDbTxnManager2 {
otherTxnMgr.closeTxnManager();
}
+ /**
+ * check that locks in Waiting state show what they are waiting on
+ * This test is somewhat abusive in that it make DbLockManager retain locks for 2
+ * different queries (which are not part of the same transaction) which can never
+ * happen in real use cases... but it makes testing convenient.
+ * @throws Exception
+ */
+ @Test
+ public void testLockBlockedBy() throws Exception {
+ CommandProcessorResponse cpr = driver.run("create table TAB_BLOCKED (a int, b int) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')");
+ checkCmdOnDriver(cpr);
+ cpr = driver.compileAndRespond("select * from TAB_BLOCKED");
+ checkCmdOnDriver(cpr);
+ txnMgr.acquireLocks(driver.getPlan(), ctx, "I AM SAM");
+ List<ShowLocksResponseElement> locks = getLocks(txnMgr);
+ Assert.assertEquals("Unexpected lock count", 1, locks.size());
+ checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "TAB_BLOCKED", null, locks.get(0));
+ cpr = driver.compileAndRespond("drop table TAB_BLOCKED");
+ checkCmdOnDriver(cpr);
+ ((DbTxnManager)txnMgr).acquireLocks(driver.getPlan(), ctx, "SAM I AM", false);//make non-blocking
+ locks = getLocks(txnMgr);
+ Assert.assertEquals("Unexpected lock count", 2, locks.size());
+ checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "TAB_BLOCKED", null, locks.get(0));
+ checkLock(LockType.EXCLUSIVE, LockState.WAITING, "default", "TAB_BLOCKED", null, locks.get(1));
+ Assert.assertEquals("BlockedByExtId doesn't match", locks.get(0).getLockid(), locks.get(1).getBlockedByExtId());
+ Assert.assertEquals("BlockedByIntId doesn't match", locks.get(0).getLockIdInternal(), locks.get(1).getBlockedByIntId());
+ }
+
@Test
public void testDummyTxnManagerOnAcidTable() throws Exception {
// Create an ACID table with DbTxnManager
http://git-wip-us.apache.org/repos/asf/hive/blob/4e9f95a1/ql/src/test/results/clientpositive/dbtxnmgr_showlocks.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/dbtxnmgr_showlocks.q.out b/ql/src/test/results/clientpositive/dbtxnmgr_showlocks.q.out
index d9d2ed6..46d8ea1 100644
--- a/ql/src/test/results/clientpositive/dbtxnmgr_showlocks.q.out
+++ b/ql/src/test/results/clientpositive/dbtxnmgr_showlocks.q.out
@@ -2,17 +2,17 @@ PREHOOK: query: show locks
PREHOOK: type: SHOWLOCKS
POSTHOOK: query: show locks
POSTHOOK: type: SHOWLOCKS
-Lock ID Database Table Partition State Type Transaction ID Last Hearbeat Acquired At User Hostname
+Lock ID Database Table Partition State Blocked By Type Transaction ID Last Hearbeat Acquired At User
PREHOOK: query: show locks extended
PREHOOK: type: SHOWLOCKS
POSTHOOK: query: show locks extended
POSTHOOK: type: SHOWLOCKS
-Lock ID Database Table Partition State Type Transaction ID Last Hearbeat Acquired At User Hostname
+Lock ID Database Table Partition State Blocked By Type Transaction ID Last Hearbeat Acquired At User
PREHOOK: query: show locks default
PREHOOK: type: SHOWLOCKS
POSTHOOK: query: show locks default
POSTHOOK: type: SHOWLOCKS
-Lock ID Database Table Partition State Type Transaction ID Last Hearbeat Acquired At User Hostname
+Lock ID Database Table Partition State Blocked By Type Transaction ID Last Hearbeat Acquired At User
PREHOOK: query: show transactions
PREHOOK: type: SHOW TRANSACTIONS
POSTHOOK: query: show transactions
[18/50] [abbrv] hive git commit: HIVE-12960: Migrate Column Stats
Extrapolation and UniformDistribution to HBaseStore (Pengcheng Xiong,
reviewed by Ashutosh Chauhan)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/96862093/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsExtrapolation.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsExtrapolation.java b/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsExtrapolation.java
new file mode 100644
index 0000000..f4e55ed
--- /dev/null
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsExtrapolation.java
@@ -0,0 +1,717 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hive.metastore.hbase;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.StatObjectConverter;
+import org.apache.hadoop.hive.metastore.api.AggrStats;
+import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.SerDeInfo;
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
+import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+public class TestHBaseAggregateStatsExtrapolation {
+ private static final Logger LOG = LoggerFactory
+ .getLogger(TestHBaseAggregateStatsExtrapolation.class.getName());
+
+ @Mock
+ HTableInterface htable;
+ private HBaseStore store;
+ SortedMap<String, Cell> rows = new TreeMap<>();
+
+ // NDV will be 3 for the bitVectors
+ String bitVectors = "{0, 4, 5, 7}{0, 1}{0, 1, 2}{0, 1, 4}{0}{0, 2}{0, 3}{0, 2, 3, 4}{0, 1, 4}{0, 1}{0}{0, 1, 3, 8}{0, 2}{0, 2}{0, 9}{0, 1, 4}";
+
+ @Before
+ public void before() throws IOException {
+ MockitoAnnotations.initMocks(this);
+ HiveConf conf = new HiveConf();
+ conf.setBoolean(HBaseReadWrite.NO_CACHE_CONF, true);
+ store = MockUtils.init(conf, htable, rows);
+ store.backdoor().getStatsCache().resetCounters();
+ }
+
+ private static interface Checker {
+ void checkStats(AggrStats aggrStats) throws Exception;
+ }
+
+ @Test
+ public void allPartitionsHaveBitVectorStatusLong() throws Exception {
+ String dbName = "default";
+ String tableName = "snp";
+ long now = System.currentTimeMillis();
+ List<FieldSchema> cols = new ArrayList<>();
+ cols.add(new FieldSchema("col1", "long", "nocomment"));
+ SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
+ StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
+ serde, null, null, Collections.<String, String> emptyMap());
+ List<FieldSchema> partCols = new ArrayList<>();
+ partCols.add(new FieldSchema("ds", "string", ""));
+ Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
+ Collections.<String, String> emptyMap(), null, null, null);
+ store.createTable(table);
+
+ List<List<String>> partVals = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ List<String> partVal = Arrays.asList("" + i);
+ partVals.add(partVal);
+ StorageDescriptor psd = new StorageDescriptor(sd);
+ psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
+ Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
+ Collections.<String, String> emptyMap());
+ store.addPartition(part);
+ ColumnStatistics cs = new ColumnStatistics();
+ ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
+ desc.setLastAnalyzed(now);
+ desc.setPartName("ds=" + partVal);
+ cs.setStatsDesc(desc);
+ ColumnStatisticsObj obj = new ColumnStatisticsObj();
+ obj.setColName("col1");
+ obj.setColType("long");
+ ColumnStatisticsData data = new ColumnStatisticsData();
+ LongColumnStatsData dcsd = new LongColumnStatsData();
+ dcsd.setHighValue(1000 + i);
+ dcsd.setLowValue(-1000 - i);
+ dcsd.setNumNulls(i);
+ dcsd.setNumDVs(10 * i + 1);
+ dcsd.setBitVectors(bitVectors);
+ data.setLongStats(dcsd);
+ obj.setStatsData(data);
+ cs.addToStatsObj(obj);
+ store.updatePartitionColumnStatistics(cs, partVal);
+ }
+
+ Checker statChecker = new Checker() {
+ @Override
+ public void checkStats(AggrStats aggrStats) throws Exception {
+ Assert.assertEquals(10, aggrStats.getPartsFound());
+ Assert.assertEquals(1, aggrStats.getColStatsSize());
+ ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
+ Assert.assertEquals("col1", cso.getColName());
+ Assert.assertEquals("long", cso.getColType());
+ LongColumnStatsData lcsd = cso.getStatsData().getLongStats();
+ Assert.assertEquals(1009, lcsd.getHighValue(), 0.01);
+ Assert.assertEquals(-1009, lcsd.getLowValue(), 0.01);
+ Assert.assertEquals(45, lcsd.getNumNulls());
+ Assert.assertEquals(3, lcsd.getNumDVs());
+ }
+ };
+ List<String> partNames = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ partNames.add("ds=" + i);
+ }
+ AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
+ Arrays.asList("col1"));
+ statChecker.checkStats(aggrStats);
+ }
+
+ @Test
+ public void allPartitionsHaveBitVectorStatusDecimal() throws Exception {
+ String dbName = "default";
+ String tableName = "snp";
+ long now = System.currentTimeMillis();
+ List<FieldSchema> cols = new ArrayList<>();
+ cols.add(new FieldSchema("col1_decimal", "decimal", "nocomment"));
+ SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
+ StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
+ serde, null, null, Collections.<String, String> emptyMap());
+ List<FieldSchema> partCols = new ArrayList<>();
+ partCols.add(new FieldSchema("ds", "string", ""));
+ Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
+ Collections.<String, String> emptyMap(), null, null, null);
+ store.createTable(table);
+
+ List<List<String>> partVals = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ List<String> partVal = Arrays.asList("" + i);
+ partVals.add(partVal);
+ StorageDescriptor psd = new StorageDescriptor(sd);
+ psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
+ Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
+ Collections.<String, String> emptyMap());
+ store.addPartition(part);
+ ColumnStatistics cs = new ColumnStatistics();
+ ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
+ desc.setLastAnalyzed(now);
+ desc.setPartName("ds=" + partVal);
+ cs.setStatsDesc(desc);
+ ColumnStatisticsObj obj = new ColumnStatisticsObj();
+ obj.setColName("col1_decimal");
+ obj.setColType("decimal");
+ ColumnStatisticsData data = new ColumnStatisticsData();
+ DecimalColumnStatsData dcsd = new DecimalColumnStatsData();
+ dcsd.setHighValue(StatObjectConverter.createThriftDecimal("" + (1000 + i)));
+ dcsd.setLowValue(StatObjectConverter.createThriftDecimal("" + (-1000 - i)));
+ dcsd.setNumNulls(i);
+ dcsd.setNumDVs(10 * i + 1);
+ dcsd.setBitVectors(bitVectors);
+ data.setDecimalStats(dcsd);
+ obj.setStatsData(data);
+ cs.addToStatsObj(obj);
+ store.updatePartitionColumnStatistics(cs, partVal);
+ }
+
+ Checker statChecker = new Checker() {
+ @Override
+ public void checkStats(AggrStats aggrStats) throws Exception {
+ Assert.assertEquals(10, aggrStats.getPartsFound());
+ Assert.assertEquals(1, aggrStats.getColStatsSize());
+ ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
+ Assert.assertEquals("col1_decimal", cso.getColName());
+ Assert.assertEquals("decimal", cso.getColType());
+ DecimalColumnStatsData lcsd = cso.getStatsData().getDecimalStats();
+ Assert.assertEquals(1009, HBaseUtils.getDoubleValue(lcsd.getHighValue()), 0.01);
+ Assert.assertEquals(-1009, HBaseUtils.getDoubleValue(lcsd.getLowValue()), 0.01);
+ Assert.assertEquals(45, lcsd.getNumNulls());
+ Assert.assertEquals(3, lcsd.getNumDVs());
+ }
+ };
+ List<String> partNames = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ partNames.add("ds=" + i);
+ }
+ AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
+ Arrays.asList("col1_decimal"));
+ statChecker.checkStats(aggrStats);
+ }
+
+ @Test
+ public void allPartitionsHaveBitVectorStatusDouble() throws Exception {
+ String dbName = "default";
+ String tableName = "snp";
+ long now = System.currentTimeMillis();
+ List<FieldSchema> cols = new ArrayList<>();
+ cols.add(new FieldSchema("col1_double", "double", "nocomment"));
+ SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
+ StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
+ serde, null, null, Collections.<String, String> emptyMap());
+ List<FieldSchema> partCols = new ArrayList<>();
+ partCols.add(new FieldSchema("ds", "string", ""));
+ Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
+ Collections.<String, String> emptyMap(), null, null, null);
+ store.createTable(table);
+
+ List<List<String>> partVals = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ List<String> partVal = Arrays.asList("" + i);
+ partVals.add(partVal);
+ StorageDescriptor psd = new StorageDescriptor(sd);
+ psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
+ Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
+ Collections.<String, String> emptyMap());
+ store.addPartition(part);
+ ColumnStatistics cs = new ColumnStatistics();
+ ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
+ desc.setLastAnalyzed(now);
+ desc.setPartName("ds=" + partVal);
+ cs.setStatsDesc(desc);
+ ColumnStatisticsObj obj = new ColumnStatisticsObj();
+ obj.setColName("col1_double");
+ obj.setColType("double");
+ ColumnStatisticsData data = new ColumnStatisticsData();
+ DoubleColumnStatsData dcsd = new DoubleColumnStatsData();
+ dcsd.setHighValue(1000 + i);
+ dcsd.setLowValue(-1000 - i);
+ dcsd.setNumNulls(i);
+ dcsd.setNumDVs(10 * i + 1);
+ dcsd.setBitVectors(bitVectors);
+ data.setDoubleStats(dcsd);
+ obj.setStatsData(data);
+ cs.addToStatsObj(obj);
+ store.updatePartitionColumnStatistics(cs, partVal);
+ }
+
+ Checker statChecker = new Checker() {
+ @Override
+ public void checkStats(AggrStats aggrStats) throws Exception {
+ Assert.assertEquals(10, aggrStats.getPartsFound());
+ Assert.assertEquals(1, aggrStats.getColStatsSize());
+ ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
+ Assert.assertEquals("col1_double", cso.getColName());
+ Assert.assertEquals("double", cso.getColType());
+ DoubleColumnStatsData lcsd = cso.getStatsData().getDoubleStats();
+ Assert.assertEquals(1009, lcsd.getHighValue(), 0.01);
+ Assert.assertEquals(-1009, lcsd.getLowValue(), 0.01);
+ Assert.assertEquals(45, lcsd.getNumNulls());
+ Assert.assertEquals(3, lcsd.getNumDVs());
+ }
+ };
+ List<String> partNames = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ partNames.add("ds=" + i);
+ }
+ AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
+ Arrays.asList("col1_double"));
+ statChecker.checkStats(aggrStats);
+ }
+
+ @Test
+ public void allPartitionsHaveBitVectorStatusString() throws Exception {
+ String dbName = "default";
+ String tableName = "snp";
+ long now = System.currentTimeMillis();
+ List<FieldSchema> cols = new ArrayList<>();
+ cols.add(new FieldSchema("col1_string", "string", "nocomment"));
+ SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
+ StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
+ serde, null, null, Collections.<String, String> emptyMap());
+ List<FieldSchema> partCols = new ArrayList<>();
+ partCols.add(new FieldSchema("ds", "string", ""));
+ Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
+ Collections.<String, String> emptyMap(), null, null, null);
+ store.createTable(table);
+
+ List<List<String>> partVals = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ List<String> partVal = Arrays.asList("" + i);
+ partVals.add(partVal);
+ StorageDescriptor psd = new StorageDescriptor(sd);
+ psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
+ Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
+ Collections.<String, String> emptyMap());
+ store.addPartition(part);
+ ColumnStatistics cs = new ColumnStatistics();
+ ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
+ desc.setLastAnalyzed(now);
+ desc.setPartName("ds=" + partVal);
+ cs.setStatsDesc(desc);
+ ColumnStatisticsObj obj = new ColumnStatisticsObj();
+ obj.setColName("col1_string");
+ obj.setColType("string");
+ ColumnStatisticsData data = new ColumnStatisticsData();
+ StringColumnStatsData dcsd = new StringColumnStatsData();
+ dcsd.setAvgColLen(i + 1);
+ dcsd.setMaxColLen(i + 10);
+ dcsd.setNumNulls(i);
+ dcsd.setNumDVs(10 * i + 1);
+ dcsd.setBitVectors(bitVectors);
+ data.setStringStats(dcsd);
+ obj.setStatsData(data);
+ cs.addToStatsObj(obj);
+ store.updatePartitionColumnStatistics(cs, partVal);
+ }
+
+ Checker statChecker = new Checker() {
+ @Override
+ public void checkStats(AggrStats aggrStats) throws Exception {
+ Assert.assertEquals(10, aggrStats.getPartsFound());
+ Assert.assertEquals(1, aggrStats.getColStatsSize());
+ ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
+ Assert.assertEquals("col1_string", cso.getColName());
+ Assert.assertEquals("string", cso.getColType());
+ StringColumnStatsData lcsd = cso.getStatsData().getStringStats();
+ Assert.assertEquals(10, lcsd.getAvgColLen(), 0.01);
+ Assert.assertEquals(19, lcsd.getMaxColLen(), 0.01);
+ Assert.assertEquals(45, lcsd.getNumNulls());
+ Assert.assertEquals(3, lcsd.getNumDVs());
+ }
+ };
+ List<String> partNames = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ partNames.add("ds=" + i);
+ }
+ AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
+ Arrays.asList("col1_string"));
+ statChecker.checkStats(aggrStats);
+ }
+
+ @Test
+ public void noPartitionsHaveBitVectorStatus() throws Exception {
+ String dbName = "default";
+ String tableName = "snp";
+ long now = System.currentTimeMillis();
+ List<FieldSchema> cols = new ArrayList<>();
+ cols.add(new FieldSchema("col2", "long", "nocomment"));
+ SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
+ StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
+ serde, null, null, Collections.<String, String> emptyMap());
+ List<FieldSchema> partCols = new ArrayList<>();
+ partCols.add(new FieldSchema("ds", "string", ""));
+ Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
+ Collections.<String, String> emptyMap(), null, null, null);
+ store.createTable(table);
+
+ List<List<String>> partVals = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ List<String> partVal = Arrays.asList("" + i);
+ partVals.add(partVal);
+ StorageDescriptor psd = new StorageDescriptor(sd);
+ psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
+ Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
+ Collections.<String, String> emptyMap());
+ store.addPartition(part);
+ ColumnStatistics cs = new ColumnStatistics();
+ ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
+ desc.setLastAnalyzed(now);
+ desc.setPartName("ds=" + partVal);
+ cs.setStatsDesc(desc);
+ ColumnStatisticsObj obj = new ColumnStatisticsObj();
+ obj.setColName("col2");
+ obj.setColType("long");
+ ColumnStatisticsData data = new ColumnStatisticsData();
+ LongColumnStatsData dcsd = new LongColumnStatsData();
+ dcsd.setHighValue(1000 + i);
+ dcsd.setLowValue(-1000 - i);
+ dcsd.setNumNulls(i);
+ dcsd.setNumDVs(10 * i);
+ data.setLongStats(dcsd);
+ obj.setStatsData(data);
+ cs.addToStatsObj(obj);
+ store.updatePartitionColumnStatistics(cs, partVal);
+ }
+
+ Checker statChecker = new Checker() {
+ @Override
+ public void checkStats(AggrStats aggrStats) throws Exception {
+ Assert.assertEquals(10, aggrStats.getPartsFound());
+ Assert.assertEquals(1, aggrStats.getColStatsSize());
+ ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
+ Assert.assertEquals("col2", cso.getColName());
+ Assert.assertEquals("long", cso.getColType());
+ LongColumnStatsData lcsd = cso.getStatsData().getLongStats();
+ Assert.assertEquals(1009, lcsd.getHighValue(), 0.01);
+ Assert.assertEquals(-1009, lcsd.getLowValue(), 0.01);
+ Assert.assertEquals(45, lcsd.getNumNulls());
+ Assert.assertEquals(90, lcsd.getNumDVs());
+ }
+ };
+ List<String> partNames = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ partNames.add("ds=" + i);
+ }
+ AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
+ Arrays.asList("col2"));
+ statChecker.checkStats(aggrStats);
+ }
+
+ @Test
+ public void TwoEndsOfPartitionsHaveBitVectorStatus() throws Exception {
+ String dbName = "default";
+ String tableName = "snp";
+ long now = System.currentTimeMillis();
+ List<FieldSchema> cols = new ArrayList<>();
+ cols.add(new FieldSchema("col3", "long", "nocomment"));
+ SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
+ StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
+ serde, null, null, Collections.<String, String> emptyMap());
+ List<FieldSchema> partCols = new ArrayList<>();
+ partCols.add(new FieldSchema("ds", "string", ""));
+ Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
+ Collections.<String, String> emptyMap(), null, null, null);
+ store.createTable(table);
+
+ List<List<String>> partVals = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ List<String> partVal = Arrays.asList("" + i);
+ partVals.add(partVal);
+ StorageDescriptor psd = new StorageDescriptor(sd);
+ psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
+ Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
+ Collections.<String, String> emptyMap());
+ store.addPartition(part);
+ if (i < 2 || i > 7) {
+ ColumnStatistics cs = new ColumnStatistics();
+ ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
+ desc.setLastAnalyzed(now);
+ desc.setPartName("ds=" + partVal);
+ cs.setStatsDesc(desc);
+ ColumnStatisticsObj obj = new ColumnStatisticsObj();
+ obj.setColName("col3");
+ obj.setColType("long");
+ ColumnStatisticsData data = new ColumnStatisticsData();
+ LongColumnStatsData dcsd = new LongColumnStatsData();
+ dcsd.setHighValue(1000 + i);
+ dcsd.setLowValue(-1000 - i);
+ dcsd.setNumNulls(i);
+ dcsd.setNumDVs(10 * i);
+ dcsd.setBitVectors(bitVectors);
+ data.setLongStats(dcsd);
+ obj.setStatsData(data);
+ cs.addToStatsObj(obj);
+ store.updatePartitionColumnStatistics(cs, partVal);
+ }
+ }
+
+ Checker statChecker = new Checker() {
+ @Override
+ public void checkStats(AggrStats aggrStats) throws Exception {
+ Assert.assertEquals(4, aggrStats.getPartsFound());
+ Assert.assertEquals(1, aggrStats.getColStatsSize());
+ ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
+ Assert.assertEquals("col3", cso.getColName());
+ Assert.assertEquals("long", cso.getColType());
+ LongColumnStatsData lcsd = cso.getStatsData().getLongStats();
+ Assert.assertEquals(1010, lcsd.getHighValue(), 0.01);
+ Assert.assertEquals(-1010, lcsd.getLowValue(), 0.01);
+ Assert.assertEquals(45, lcsd.getNumNulls());
+ Assert.assertEquals(3, lcsd.getNumDVs());
+ }
+ };
+ List<String> partNames = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ partNames.add("ds=" + i);
+ }
+ AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
+ Arrays.asList("col3"));
+ statChecker.checkStats(aggrStats);
+ }
+
+ @Test
+ public void MiddleOfPartitionsHaveBitVectorStatus() throws Exception {
+ String dbName = "default";
+ String tableName = "snp";
+ long now = System.currentTimeMillis();
+ List<FieldSchema> cols = new ArrayList<>();
+ cols.add(new FieldSchema("col4", "long", "nocomment"));
+ SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
+ StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
+ serde, null, null, Collections.<String, String> emptyMap());
+ List<FieldSchema> partCols = new ArrayList<>();
+ partCols.add(new FieldSchema("ds", "string", ""));
+ Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
+ Collections.<String, String> emptyMap(), null, null, null);
+ store.createTable(table);
+
+ List<List<String>> partVals = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ List<String> partVal = Arrays.asList("" + i);
+ partVals.add(partVal);
+ StorageDescriptor psd = new StorageDescriptor(sd);
+ psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
+ Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
+ Collections.<String, String> emptyMap());
+ store.addPartition(part);
+ if (i > 2 && i < 7) {
+ ColumnStatistics cs = new ColumnStatistics();
+ ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
+ desc.setLastAnalyzed(now);
+ desc.setPartName("ds=" + partVal);
+ cs.setStatsDesc(desc);
+ ColumnStatisticsObj obj = new ColumnStatisticsObj();
+ obj.setColName("col4");
+ obj.setColType("long");
+ ColumnStatisticsData data = new ColumnStatisticsData();
+ LongColumnStatsData dcsd = new LongColumnStatsData();
+ dcsd.setHighValue(1000 + i);
+ dcsd.setLowValue(-1000 - i);
+ dcsd.setNumNulls(i);
+ dcsd.setNumDVs(10 * i);
+ dcsd.setBitVectors(bitVectors);
+ data.setLongStats(dcsd);
+ obj.setStatsData(data);
+ cs.addToStatsObj(obj);
+ store.updatePartitionColumnStatistics(cs, partVal);
+ }
+ }
+
+ Checker statChecker = new Checker() {
+ @Override
+ public void checkStats(AggrStats aggrStats) throws Exception {
+ Assert.assertEquals(4, aggrStats.getPartsFound());
+ Assert.assertEquals(1, aggrStats.getColStatsSize());
+ ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
+ Assert.assertEquals("col4", cso.getColName());
+ Assert.assertEquals("long", cso.getColType());
+ LongColumnStatsData lcsd = cso.getStatsData().getLongStats();
+ Assert.assertEquals(1006, lcsd.getHighValue(), 0.01);
+ Assert.assertEquals(-1006, lcsd.getLowValue(), 0.01);
+ Assert.assertEquals(45, lcsd.getNumNulls());
+ Assert.assertEquals(3, lcsd.getNumDVs());
+ }
+ };
+ List<String> partNames = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ partNames.add("ds=" + i);
+ }
+ AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
+ Arrays.asList("col4"));
+ statChecker.checkStats(aggrStats);
+ }
+
+ @Test
+ public void TwoEndsAndMiddleOfPartitionsHaveBitVectorStatusLong() throws Exception {
+ String dbName = "default";
+ String tableName = "snp";
+ long now = System.currentTimeMillis();
+ List<FieldSchema> cols = new ArrayList<>();
+ cols.add(new FieldSchema("col5", "long", "nocomment"));
+ SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
+ StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
+ serde, null, null, Collections.<String, String> emptyMap());
+ List<FieldSchema> partCols = new ArrayList<>();
+ partCols.add(new FieldSchema("ds", "string", ""));
+ Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
+ Collections.<String, String> emptyMap(), null, null, null);
+ store.createTable(table);
+
+ List<List<String>> partVals = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ List<String> partVal = Arrays.asList("" + i);
+ partVals.add(partVal);
+ StorageDescriptor psd = new StorageDescriptor(sd);
+ psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
+ Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
+ Collections.<String, String> emptyMap());
+ store.addPartition(part);
+ if (i == 0 || i == 2 || i == 3 || i == 5 || i == 6 || i == 8) {
+ ColumnStatistics cs = new ColumnStatistics();
+ ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
+ desc.setLastAnalyzed(now);
+ desc.setPartName("ds=" + partVal);
+ cs.setStatsDesc(desc);
+ ColumnStatisticsObj obj = new ColumnStatisticsObj();
+ obj.setColName("col5");
+ obj.setColType("long");
+ ColumnStatisticsData data = new ColumnStatisticsData();
+ LongColumnStatsData dcsd = new LongColumnStatsData();
+ dcsd.setHighValue(1000 + i);
+ dcsd.setLowValue(-1000 - i);
+ dcsd.setNumNulls(i);
+ dcsd.setNumDVs(10 * i);
+ dcsd.setBitVectors(bitVectors);
+ data.setLongStats(dcsd);
+ obj.setStatsData(data);
+ cs.addToStatsObj(obj);
+ store.updatePartitionColumnStatistics(cs, partVal);
+ }
+ }
+
+ Checker statChecker = new Checker() {
+ @Override
+ public void checkStats(AggrStats aggrStats) throws Exception {
+ Assert.assertEquals(6, aggrStats.getPartsFound());
+ Assert.assertEquals(1, aggrStats.getColStatsSize());
+ ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
+ Assert.assertEquals("col5", cso.getColName());
+ Assert.assertEquals("long", cso.getColType());
+ LongColumnStatsData lcsd = cso.getStatsData().getLongStats();
+ Assert.assertEquals(1010, lcsd.getHighValue(), 0.01);
+ Assert.assertEquals(-1010, lcsd.getLowValue(), 0.01);
+ Assert.assertEquals(40, lcsd.getNumNulls());
+ Assert.assertEquals(3, lcsd.getNumDVs());
+ }
+ };
+ List<String> partNames = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ partNames.add("ds=" + i);
+ }
+ AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
+ Arrays.asList("col5"));
+ statChecker.checkStats(aggrStats);
+ }
+
+ @Test
+ public void TwoEndsAndMiddleOfPartitionsHaveBitVectorStatusDouble() throws Exception {
+ String dbName = "default";
+ String tableName = "snp";
+ long now = System.currentTimeMillis();
+ List<FieldSchema> cols = new ArrayList<>();
+ cols.add(new FieldSchema("col5_double", "double", "nocomment"));
+ SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
+ StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
+ serde, null, null, Collections.<String, String> emptyMap());
+ List<FieldSchema> partCols = new ArrayList<>();
+ partCols.add(new FieldSchema("ds", "string", ""));
+ Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
+ Collections.<String, String> emptyMap(), null, null, null);
+ store.createTable(table);
+
+ List<List<String>> partVals = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ List<String> partVal = Arrays.asList("" + i);
+ partVals.add(partVal);
+ StorageDescriptor psd = new StorageDescriptor(sd);
+ psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
+ Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
+ Collections.<String, String> emptyMap());
+ store.addPartition(part);
+ if (i == 0 || i == 2 || i == 3 || i == 5 || i == 6 || i == 8) {
+ ColumnStatistics cs = new ColumnStatistics();
+ ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
+ desc.setLastAnalyzed(now);
+ desc.setPartName("ds=" + partVal);
+ cs.setStatsDesc(desc);
+ ColumnStatisticsObj obj = new ColumnStatisticsObj();
+ obj.setColName("col5_double");
+ obj.setColType("double");
+ ColumnStatisticsData data = new ColumnStatisticsData();
+ DoubleColumnStatsData dcsd = new DoubleColumnStatsData();
+ dcsd.setHighValue(1000 + i);
+ dcsd.setLowValue(-1000 - i);
+ dcsd.setNumNulls(i);
+ dcsd.setNumDVs(10 * i);
+ dcsd.setBitVectors(bitVectors);
+ data.setDoubleStats(dcsd);
+ obj.setStatsData(data);
+ cs.addToStatsObj(obj);
+ store.updatePartitionColumnStatistics(cs, partVal);
+ }
+ }
+
+ Checker statChecker = new Checker() {
+ @Override
+ public void checkStats(AggrStats aggrStats) throws Exception {
+ Assert.assertEquals(6, aggrStats.getPartsFound());
+ Assert.assertEquals(1, aggrStats.getColStatsSize());
+ ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
+ Assert.assertEquals("col5_double", cso.getColName());
+ Assert.assertEquals("double", cso.getColType());
+ DoubleColumnStatsData lcsd = cso.getStatsData().getDoubleStats();
+ Assert.assertEquals(1010, lcsd.getHighValue(), 0.01);
+ Assert.assertEquals(-1010, lcsd.getLowValue(), 0.01);
+ Assert.assertEquals(40, lcsd.getNumNulls());
+ Assert.assertEquals(3, lcsd.getNumDVs());
+ }
+ };
+ List<String> partNames = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ partNames.add("ds=" + i);
+ }
+ AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
+ Arrays.asList("col5_double"));
+ statChecker.checkStats(aggrStats);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/96862093/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsNDVUniformDist.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsNDVUniformDist.java b/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsNDVUniformDist.java
new file mode 100644
index 0000000..62918be
--- /dev/null
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsNDVUniformDist.java
@@ -0,0 +1,581 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hive.metastore.hbase;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.StatObjectConverter;
+import org.apache.hadoop.hive.metastore.api.AggrStats;
+import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.SerDeInfo;
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+public class TestHBaseAggregateStatsNDVUniformDist {
+ private static final Logger LOG = LoggerFactory
+ .getLogger(TestHBaseAggregateStatsNDVUniformDist.class.getName());
+
+ @Mock
+ HTableInterface htable;
+ private HBaseStore store;
+ SortedMap<String, Cell> rows = new TreeMap<>();
+
+ // NDV will be 3 for bitVectors[0] and 12 for bitVectors[1]
+ String bitVectors[] = {
+ "{0, 4, 5, 7}{0, 1}{0, 1, 2}{0, 1, 4}{0}{0, 2}{0, 3}{0, 2, 3, 4}{0, 1, 4}{0, 1}{0}{0, 1, 3, 8}{0, 2}{0, 2}{0, 9}{0, 1, 4}",
+ "{1, 2}{1, 2}{1, 2}{1, 2}{1, 2}{1, 2}{1, 2}{1, 2}{1, 2}{1, 2}{1, 2}{1, 2}{1, 2}{1, 2}{1, 2}{1, 2}" };
+
+ @Before
+ public void before() throws IOException {
+ MockitoAnnotations.initMocks(this);
+ HiveConf conf = new HiveConf();
+ conf.setBoolean(HBaseReadWrite.NO_CACHE_CONF, true);
+ conf.setBoolean(HiveConf.ConfVars.HIVE_METASTORE_STATS_NDV_DENSITY_FUNCTION.varname, true);
+ store = MockUtils.init(conf, htable, rows);
+ store.backdoor().getStatsCache().resetCounters();
+ }
+
+ private static interface Checker {
+ void checkStats(AggrStats aggrStats) throws Exception;
+ }
+
+ @Test
+ public void allPartitionsHaveBitVectorStatus() throws Exception {
+ String dbName = "default";
+ String tableName = "snp";
+ long now = System.currentTimeMillis();
+ List<FieldSchema> cols = new ArrayList<>();
+ cols.add(new FieldSchema("col1", "long", "nocomment"));
+ SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
+ StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
+ serde, null, null, Collections.<String, String> emptyMap());
+ List<FieldSchema> partCols = new ArrayList<>();
+ partCols.add(new FieldSchema("ds", "string", ""));
+ Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
+ Collections.<String, String> emptyMap(), null, null, null);
+ store.createTable(table);
+
+ List<List<String>> partVals = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ List<String> partVal = Arrays.asList("" + i);
+ partVals.add(partVal);
+ StorageDescriptor psd = new StorageDescriptor(sd);
+ psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
+ Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
+ Collections.<String, String> emptyMap());
+ store.addPartition(part);
+ ColumnStatistics cs = new ColumnStatistics();
+ ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
+ desc.setLastAnalyzed(now);
+ desc.setPartName("ds=" + partVal);
+ cs.setStatsDesc(desc);
+ ColumnStatisticsObj obj = new ColumnStatisticsObj();
+ obj.setColName("col1");
+ obj.setColType("long");
+ ColumnStatisticsData data = new ColumnStatisticsData();
+ LongColumnStatsData dcsd = new LongColumnStatsData();
+ dcsd.setHighValue(1000 + i);
+ dcsd.setLowValue(-1000 - i);
+ dcsd.setNumNulls(i);
+ dcsd.setNumDVs(10 * i + 1);
+ dcsd.setBitVectors(bitVectors[0]);
+ data.setLongStats(dcsd);
+ obj.setStatsData(data);
+ cs.addToStatsObj(obj);
+ store.updatePartitionColumnStatistics(cs, partVal);
+ }
+
+ Checker statChecker = new Checker() {
+ @Override
+ public void checkStats(AggrStats aggrStats) throws Exception {
+ Assert.assertEquals(10, aggrStats.getPartsFound());
+ Assert.assertEquals(1, aggrStats.getColStatsSize());
+ ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
+ Assert.assertEquals("col1", cso.getColName());
+ Assert.assertEquals("long", cso.getColType());
+ LongColumnStatsData lcsd = cso.getStatsData().getLongStats();
+ Assert.assertEquals(1009, lcsd.getHighValue(), 0.01);
+ Assert.assertEquals(-1009, lcsd.getLowValue(), 0.01);
+ Assert.assertEquals(45, lcsd.getNumNulls());
+ Assert.assertEquals(3, lcsd.getNumDVs());
+ }
+ };
+ List<String> partNames = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ partNames.add("ds=" + i);
+ }
+ AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
+ Arrays.asList("col1"));
+ statChecker.checkStats(aggrStats);
+ }
+
+ @Test
+ public void noPartitionsHaveBitVectorStatus() throws Exception {
+ String dbName = "default";
+ String tableName = "snp";
+ long now = System.currentTimeMillis();
+ List<FieldSchema> cols = new ArrayList<>();
+ cols.add(new FieldSchema("col2", "long", "nocomment"));
+ SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
+ StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
+ serde, null, null, Collections.<String, String> emptyMap());
+ List<FieldSchema> partCols = new ArrayList<>();
+ partCols.add(new FieldSchema("ds", "string", ""));
+ Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
+ Collections.<String, String> emptyMap(), null, null, null);
+ store.createTable(table);
+
+ List<List<String>> partVals = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ List<String> partVal = Arrays.asList("" + i);
+ partVals.add(partVal);
+ StorageDescriptor psd = new StorageDescriptor(sd);
+ psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
+ Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
+ Collections.<String, String> emptyMap());
+ store.addPartition(part);
+ ColumnStatistics cs = new ColumnStatistics();
+ ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
+ desc.setLastAnalyzed(now);
+ desc.setPartName("ds=" + partVal);
+ cs.setStatsDesc(desc);
+ ColumnStatisticsObj obj = new ColumnStatisticsObj();
+ obj.setColName("col2");
+ obj.setColType("long");
+ ColumnStatisticsData data = new ColumnStatisticsData();
+ LongColumnStatsData dcsd = new LongColumnStatsData();
+ dcsd.setHighValue(1000 + i);
+ dcsd.setLowValue(-1000 - i);
+ dcsd.setNumNulls(i);
+ dcsd.setNumDVs(10 * i + 1);
+ data.setLongStats(dcsd);
+ obj.setStatsData(data);
+ cs.addToStatsObj(obj);
+ store.updatePartitionColumnStatistics(cs, partVal);
+ }
+
+ Checker statChecker = new Checker() {
+ @Override
+ public void checkStats(AggrStats aggrStats) throws Exception {
+ Assert.assertEquals(10, aggrStats.getPartsFound());
+ Assert.assertEquals(1, aggrStats.getColStatsSize());
+ ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
+ Assert.assertEquals("col2", cso.getColName());
+ Assert.assertEquals("long", cso.getColType());
+ LongColumnStatsData lcsd = cso.getStatsData().getLongStats();
+ Assert.assertEquals(1009, lcsd.getHighValue(), 0.01);
+ Assert.assertEquals(-1009, lcsd.getLowValue(), 0.01);
+ Assert.assertEquals(45, lcsd.getNumNulls());
+ Assert.assertEquals(91, lcsd.getNumDVs());
+ }
+ };
+ List<String> partNames = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ partNames.add("ds=" + i);
+ }
+ AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
+ Arrays.asList("col2"));
+ statChecker.checkStats(aggrStats);
+ }
+
+ @Test
+ public void TwoEndsOfPartitionsHaveBitVectorStatus() throws Exception {
+ String dbName = "default";
+ String tableName = "snp";
+ long now = System.currentTimeMillis();
+ List<FieldSchema> cols = new ArrayList<>();
+ cols.add(new FieldSchema("col3", "long", "nocomment"));
+ SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
+ StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
+ serde, null, null, Collections.<String, String> emptyMap());
+ List<FieldSchema> partCols = new ArrayList<>();
+ partCols.add(new FieldSchema("ds", "string", ""));
+ Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
+ Collections.<String, String> emptyMap(), null, null, null);
+ store.createTable(table);
+
+ List<List<String>> partVals = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ List<String> partVal = Arrays.asList("" + i);
+ partVals.add(partVal);
+ StorageDescriptor psd = new StorageDescriptor(sd);
+ psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
+ Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
+ Collections.<String, String> emptyMap());
+ store.addPartition(part);
+ if (i < 2 || i > 7) {
+ ColumnStatistics cs = new ColumnStatistics();
+ ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
+ desc.setLastAnalyzed(now);
+ desc.setPartName("ds=" + partVal);
+ cs.setStatsDesc(desc);
+ ColumnStatisticsObj obj = new ColumnStatisticsObj();
+ obj.setColName("col3");
+ obj.setColType("long");
+ ColumnStatisticsData data = new ColumnStatisticsData();
+ LongColumnStatsData dcsd = new LongColumnStatsData();
+ dcsd.setHighValue(1000 + i);
+ dcsd.setLowValue(-1000 - i);
+ dcsd.setNumNulls(i);
+ dcsd.setNumDVs(10 * i + 1);
+ dcsd.setBitVectors(bitVectors[i / 5]);
+ data.setLongStats(dcsd);
+ obj.setStatsData(data);
+ cs.addToStatsObj(obj);
+ store.updatePartitionColumnStatistics(cs, partVal);
+ }
+ }
+
+ Checker statChecker = new Checker() {
+ @Override
+ public void checkStats(AggrStats aggrStats) throws Exception {
+ Assert.assertEquals(4, aggrStats.getPartsFound());
+ Assert.assertEquals(1, aggrStats.getColStatsSize());
+ ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
+ Assert.assertEquals("col3", cso.getColName());
+ Assert.assertEquals("long", cso.getColType());
+ LongColumnStatsData lcsd = cso.getStatsData().getLongStats();
+ Assert.assertEquals(1010, lcsd.getHighValue(), 0.01);
+ Assert.assertEquals(-1010, lcsd.getLowValue(), 0.01);
+ Assert.assertEquals(45, lcsd.getNumNulls());
+ Assert.assertEquals(12, lcsd.getNumDVs());
+ }
+ };
+ List<String> partNames = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ partNames.add("ds=" + i);
+ }
+ AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
+ Arrays.asList("col3"));
+ statChecker.checkStats(aggrStats);
+ }
+
+ @Test
+ public void MiddleOfPartitionsHaveBitVectorStatus() throws Exception {
+ String dbName = "default";
+ String tableName = "snp";
+ long now = System.currentTimeMillis();
+ List<FieldSchema> cols = new ArrayList<>();
+ cols.add(new FieldSchema("col4", "long", "nocomment"));
+ SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
+ StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
+ serde, null, null, Collections.<String, String> emptyMap());
+ List<FieldSchema> partCols = new ArrayList<>();
+ partCols.add(new FieldSchema("ds", "string", ""));
+ Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
+ Collections.<String, String> emptyMap(), null, null, null);
+ store.createTable(table);
+
+ List<List<String>> partVals = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ List<String> partVal = Arrays.asList("" + i);
+ partVals.add(partVal);
+ StorageDescriptor psd = new StorageDescriptor(sd);
+ psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
+ Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
+ Collections.<String, String> emptyMap());
+ store.addPartition(part);
+ if (i > 2 && i < 7) {
+ ColumnStatistics cs = new ColumnStatistics();
+ ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
+ desc.setLastAnalyzed(now);
+ desc.setPartName("ds=" + partVal);
+ cs.setStatsDesc(desc);
+ ColumnStatisticsObj obj = new ColumnStatisticsObj();
+ obj.setColName("col4");
+ obj.setColType("long");
+ ColumnStatisticsData data = new ColumnStatisticsData();
+ LongColumnStatsData dcsd = new LongColumnStatsData();
+ dcsd.setHighValue(1000 + i);
+ dcsd.setLowValue(-1000 - i);
+ dcsd.setNumNulls(i);
+ dcsd.setNumDVs(10 * i + 1);
+ dcsd.setBitVectors(bitVectors[0]);
+ data.setLongStats(dcsd);
+ obj.setStatsData(data);
+ cs.addToStatsObj(obj);
+ store.updatePartitionColumnStatistics(cs, partVal);
+ }
+ }
+
+ Checker statChecker = new Checker() {
+ @Override
+ public void checkStats(AggrStats aggrStats) throws Exception {
+ Assert.assertEquals(4, aggrStats.getPartsFound());
+ Assert.assertEquals(1, aggrStats.getColStatsSize());
+ ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
+ Assert.assertEquals("col4", cso.getColName());
+ Assert.assertEquals("long", cso.getColType());
+ LongColumnStatsData lcsd = cso.getStatsData().getLongStats();
+ Assert.assertEquals(1006, lcsd.getHighValue(), 0.01);
+ Assert.assertEquals(-1006, lcsd.getLowValue(), 0.01);
+ Assert.assertEquals(45, lcsd.getNumNulls());
+ Assert.assertEquals(3, lcsd.getNumDVs());
+ }
+ };
+ List<String> partNames = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ partNames.add("ds=" + i);
+ }
+ AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
+ Arrays.asList("col4"));
+ statChecker.checkStats(aggrStats);
+ }
+
+ @Test
+ public void TwoEndsAndMiddleOfPartitionsHaveBitVectorStatusLong() throws Exception {
+ String dbName = "default";
+ String tableName = "snp";
+ long now = System.currentTimeMillis();
+ List<FieldSchema> cols = new ArrayList<>();
+ cols.add(new FieldSchema("col5_long", "long", "nocomment"));
+ SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
+ StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
+ serde, null, null, Collections.<String, String> emptyMap());
+ List<FieldSchema> partCols = new ArrayList<>();
+ partCols.add(new FieldSchema("ds", "string", ""));
+ Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
+ Collections.<String, String> emptyMap(), null, null, null);
+ store.createTable(table);
+
+ List<List<String>> partVals = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ List<String> partVal = Arrays.asList("" + i);
+ partVals.add(partVal);
+ StorageDescriptor psd = new StorageDescriptor(sd);
+ psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
+ Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
+ Collections.<String, String> emptyMap());
+ store.addPartition(part);
+ if (i == 0 || i == 2 || i == 3 || i == 5 || i == 6 || i == 8) {
+ ColumnStatistics cs = new ColumnStatistics();
+ ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
+ desc.setLastAnalyzed(now);
+ desc.setPartName("ds=" + partVal);
+ cs.setStatsDesc(desc);
+ ColumnStatisticsObj obj = new ColumnStatisticsObj();
+ obj.setColName("col5_long");
+ obj.setColType("long");
+ ColumnStatisticsData data = new ColumnStatisticsData();
+ LongColumnStatsData dcsd = new LongColumnStatsData();
+ dcsd.setHighValue(1000 + i);
+ dcsd.setLowValue(-1000 - i);
+ dcsd.setNumNulls(i);
+ dcsd.setNumDVs(10 * i + 1);
+ dcsd.setBitVectors(bitVectors[i / 5]);
+ data.setLongStats(dcsd);
+ obj.setStatsData(data);
+ cs.addToStatsObj(obj);
+ store.updatePartitionColumnStatistics(cs, partVal);
+ }
+ }
+
+ Checker statChecker = new Checker() {
+ @Override
+ public void checkStats(AggrStats aggrStats) throws Exception {
+ Assert.assertEquals(6, aggrStats.getPartsFound());
+ Assert.assertEquals(1, aggrStats.getColStatsSize());
+ ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
+ Assert.assertEquals("col5_long", cso.getColName());
+ Assert.assertEquals("long", cso.getColType());
+ LongColumnStatsData lcsd = cso.getStatsData().getLongStats();
+ Assert.assertEquals(1010, lcsd.getHighValue(), 0.01);
+ Assert.assertEquals(-1010, lcsd.getLowValue(), 0.01);
+ Assert.assertEquals(40, lcsd.getNumNulls());
+ Assert.assertEquals(12, lcsd.getNumDVs());
+ }
+ };
+ List<String> partNames = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ partNames.add("ds=" + i);
+ }
+ AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
+ Arrays.asList("col5_long"));
+ statChecker.checkStats(aggrStats);
+ }
+
+ @Test
+ public void TwoEndsAndMiddleOfPartitionsHaveBitVectorStatusDecimal() throws Exception {
+ String dbName = "default";
+ String tableName = "snp";
+ long now = System.currentTimeMillis();
+ List<FieldSchema> cols = new ArrayList<>();
+ cols.add(new FieldSchema("col5_decimal", "decimal", "nocomment"));
+ SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
+ StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
+ serde, null, null, Collections.<String, String> emptyMap());
+ List<FieldSchema> partCols = new ArrayList<>();
+ partCols.add(new FieldSchema("ds", "string", ""));
+ Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
+ Collections.<String, String> emptyMap(), null, null, null);
+ store.createTable(table);
+
+ List<List<String>> partVals = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ List<String> partVal = Arrays.asList("" + i);
+ partVals.add(partVal);
+ StorageDescriptor psd = new StorageDescriptor(sd);
+ psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
+ Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
+ Collections.<String, String> emptyMap());
+ store.addPartition(part);
+ if (i == 0 || i == 2 || i == 3 || i == 5 || i == 6 || i == 8) {
+ ColumnStatistics cs = new ColumnStatistics();
+ ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
+ desc.setLastAnalyzed(now);
+ desc.setPartName("ds=" + partVal);
+ cs.setStatsDesc(desc);
+ ColumnStatisticsObj obj = new ColumnStatisticsObj();
+ obj.setColName("col5_decimal");
+ obj.setColType("decimal");
+ ColumnStatisticsData data = new ColumnStatisticsData();
+ DecimalColumnStatsData dcsd = new DecimalColumnStatsData();
+ dcsd.setHighValue(StatObjectConverter.createThriftDecimal("" + (1000 + i)));
+ dcsd.setLowValue(StatObjectConverter.createThriftDecimal("" + (-1000 - i)));
+ dcsd.setNumNulls(i);
+ dcsd.setNumDVs(10 * i + 1);
+ dcsd.setBitVectors(bitVectors[i / 5]);
+ data.setDecimalStats(dcsd);
+ obj.setStatsData(data);
+ cs.addToStatsObj(obj);
+ store.updatePartitionColumnStatistics(cs, partVal);
+ }
+ }
+
+ Checker statChecker = new Checker() {
+ @Override
+ public void checkStats(AggrStats aggrStats) throws Exception {
+ Assert.assertEquals(6, aggrStats.getPartsFound());
+ Assert.assertEquals(1, aggrStats.getColStatsSize());
+ ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
+ Assert.assertEquals("col5_decimal", cso.getColName());
+ Assert.assertEquals("decimal", cso.getColType());
+ DecimalColumnStatsData lcsd = cso.getStatsData().getDecimalStats();
+ Assert.assertEquals(1010, HBaseUtils.getDoubleValue(lcsd.getHighValue()), 0.01);
+ Assert.assertEquals(-1010, HBaseUtils.getDoubleValue(lcsd.getLowValue()), 0.01);
+ Assert.assertEquals(40, lcsd.getNumNulls());
+ Assert.assertEquals(12, lcsd.getNumDVs());
+ }
+ };
+ List<String> partNames = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ partNames.add("ds=" + i);
+ }
+ AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
+ Arrays.asList("col5_decimal"));
+ statChecker.checkStats(aggrStats);
+ }
+
+ @Test
+ public void TwoEndsAndMiddleOfPartitionsHaveBitVectorStatusDouble() throws Exception {
+ String dbName = "default";
+ String tableName = "snp";
+ long now = System.currentTimeMillis();
+ List<FieldSchema> cols = new ArrayList<>();
+ cols.add(new FieldSchema("col5_double", "double", "nocomment"));
+ SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
+ StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
+ serde, null, null, Collections.<String, String> emptyMap());
+ List<FieldSchema> partCols = new ArrayList<>();
+ partCols.add(new FieldSchema("ds", "string", ""));
+ Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
+ Collections.<String, String> emptyMap(), null, null, null);
+ store.createTable(table);
+
+ List<List<String>> partVals = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ List<String> partVal = Arrays.asList("" + i);
+ partVals.add(partVal);
+ StorageDescriptor psd = new StorageDescriptor(sd);
+ psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
+ Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
+ Collections.<String, String> emptyMap());
+ store.addPartition(part);
+ if (i == 0 || i == 2 || i == 3 || i == 5 || i == 6 || i == 8) {
+ ColumnStatistics cs = new ColumnStatistics();
+ ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
+ desc.setLastAnalyzed(now);
+ desc.setPartName("ds=" + partVal);
+ cs.setStatsDesc(desc);
+ ColumnStatisticsObj obj = new ColumnStatisticsObj();
+ obj.setColName("col5_double");
+ obj.setColType("double");
+ ColumnStatisticsData data = new ColumnStatisticsData();
+ DoubleColumnStatsData dcsd = new DoubleColumnStatsData();
+ dcsd.setHighValue(1000 + i);
+ dcsd.setLowValue(-1000 - i);
+ dcsd.setNumNulls(i);
+ dcsd.setNumDVs(10 * i + 1);
+ dcsd.setBitVectors(bitVectors[i / 5]);
+ data.setDoubleStats(dcsd);
+ obj.setStatsData(data);
+ cs.addToStatsObj(obj);
+ store.updatePartitionColumnStatistics(cs, partVal);
+ }
+ }
+
+ Checker statChecker = new Checker() {
+ @Override
+ public void checkStats(AggrStats aggrStats) throws Exception {
+ Assert.assertEquals(6, aggrStats.getPartsFound());
+ Assert.assertEquals(1, aggrStats.getColStatsSize());
+ ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
+ Assert.assertEquals("col5_double", cso.getColName());
+ Assert.assertEquals("double", cso.getColType());
+ DoubleColumnStatsData lcsd = cso.getStatsData().getDoubleStats();
+ Assert.assertEquals(1010, lcsd.getHighValue(), 0.01);
+ Assert.assertEquals(-1010, lcsd.getLowValue(), 0.01);
+ Assert.assertEquals(40, lcsd.getNumNulls());
+ Assert.assertEquals(12, lcsd.getNumDVs());
+ }
+ };
+ List<String> partNames = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ partNames.add("ds=" + i);
+ }
+ AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
+ Arrays.asList("col5_double"));
+ statChecker.checkStats(aggrStats);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/96862093/ql/src/test/results/clientpositive/tez/explainuser_1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/explainuser_1.q.out b/ql/src/test/results/clientpositive/tez/explainuser_1.q.out
index b501f97..0eb9132 100644
--- a/ql/src/test/results/clientpositive/tez/explainuser_1.q.out
+++ b/ql/src/test/results/clientpositive/tez/explainuser_1.q.out
@@ -426,9 +426,9 @@ Stage-0
<-Map 8 [SIMPLE_EDGE]
SHUFFLE [RS_15]
PartitionCols:_col0, _col1, _col2
- Group By Operator [GBY_14] (rows=1 width=101)
+ Group By Operator [GBY_14] (rows=2 width=101)
Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float
- Filter Operator [FIL_49] (rows=3 width=93)
+ Filter Operator [FIL_49] (rows=5 width=74)
predicate:((((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0))) and key is not null)
TableScan [TS_11] (rows=20 width=83)
default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
@@ -446,9 +446,9 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_4]
PartitionCols:_col0, _col1, _col2
- Group By Operator [GBY_3] (rows=1 width=101)
+ Group By Operator [GBY_3] (rows=2 width=101)
Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float
- Filter Operator [FIL_48] (rows=3 width=93)
+ Filter Operator [FIL_48] (rows=5 width=74)
predicate:((((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0))) and key is not null)
TableScan [TS_0] (rows=20 width=83)
default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
@@ -1201,11 +1201,11 @@ Stage-0
Stage-1
Reducer 3
File Output Operator [FS_19]
- Select Operator [SEL_18] (rows=21 width=101)
+ Select Operator [SEL_18] (rows=36 width=101)
Output:["_col0","_col1","_col2","_col3","_col4"]
- Filter Operator [FIL_17] (rows=21 width=101)
+ Filter Operator [FIL_17] (rows=36 width=101)
predicate:((_col1 > 0) or (_col6 >= 0))
- Merge Join Operator [MERGEJOIN_28] (rows=21 width=101)
+ Merge Join Operator [MERGEJOIN_28] (rows=36 width=101)
Conds:RS_14._col0=RS_15._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col6"]
<-Map 5 [SIMPLE_EDGE]
SHUFFLE [RS_15]
@@ -1219,25 +1219,25 @@ Stage-0
<-Reducer 2 [SIMPLE_EDGE]
SHUFFLE [RS_14]
PartitionCols:_col0
- Filter Operator [FIL_9] (rows=6 width=182)
+ Filter Operator [FIL_9] (rows=10 width=182)
predicate:(((_col1 + _col4) = 2) and ((_col4 + 1) = 2))
- Merge Join Operator [MERGEJOIN_27] (rows=25 width=182)
+ Merge Join Operator [MERGEJOIN_27] (rows=40 width=182)
Conds:RS_6._col0=RS_7._col0(Left Outer),Output:["_col0","_col1","_col2","_col3","_col4"]
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_6]
PartitionCols:_col0
- Select Operator [SEL_2] (rows=5 width=74)
+ Select Operator [SEL_2] (rows=9 width=82)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_24] (rows=5 width=74)
+ Filter Operator [FIL_24] (rows=9 width=82)
predicate:((((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0))) and key is not null)
TableScan [TS_0] (rows=20 width=83)
default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
<-Map 4 [SIMPLE_EDGE]
SHUFFLE [RS_7]
PartitionCols:_col0
- Select Operator [SEL_5] (rows=5 width=71)
+ Select Operator [SEL_5] (rows=9 width=79)
Output:["_col0","_col1"]
- Filter Operator [FIL_25] (rows=5 width=74)
+ Filter Operator [FIL_25] (rows=9 width=82)
predicate:((((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0))) and key is not null)
TableScan [TS_3] (rows=20 width=83)
default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
@@ -1257,27 +1257,27 @@ Stage-0
Stage-1
Reducer 2
File Output Operator [FS_14]
- Select Operator [SEL_13] (rows=24 width=101)
+ Select Operator [SEL_13] (rows=50 width=101)
Output:["_col0","_col1","_col2","_col3","_col4"]
- Filter Operator [FIL_12] (rows=24 width=101)
+ Filter Operator [FIL_12] (rows=50 width=101)
predicate:(((_col1 + _col4) = 2) and ((_col1 > 0) or (_col6 >= 0)) and ((_col4 + 1) = 2))
- Merge Join Operator [MERGEJOIN_19] (rows=72 width=101)
+ Merge Join Operator [MERGEJOIN_19] (rows=200 width=101)
Conds:RS_8._col0=RS_9._col0(Right Outer),RS_8._col0=RS_10._col0(Right Outer),Output:["_col1","_col2","_col3","_col4","_col6"]
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_8]
PartitionCols:_col0
- Select Operator [SEL_2] (rows=6 width=77)
+ Select Operator [SEL_2] (rows=10 width=83)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_17] (rows=6 width=77)
+ Filter Operator [FIL_17] (rows=10 width=83)
predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0)))
TableScan [TS_0] (rows=20 width=83)
default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
<-Map 3 [SIMPLE_EDGE]
SHUFFLE [RS_9]
PartitionCols:_col0
- Select Operator [SEL_5] (rows=6 width=74)
+ Select Operator [SEL_5] (rows=10 width=80)
Output:["_col0","_col1"]
- Filter Operator [FIL_18] (rows=6 width=77)
+ Filter Operator [FIL_18] (rows=10 width=83)
predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0)))
TableScan [TS_3] (rows=20 width=83)
default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
@@ -1509,53 +1509,53 @@ Stage-0
Output:["_col0","_col1","_col2"]
Filter Operator [FIL_31] (rows=1 width=101)
predicate:((_col1 + _col4) >= 0)
- Merge Join Operator [MERGEJOIN_60] (rows=1 width=101)
+ Merge Join Operator [MERGEJOIN_60] (rows=2 width=101)
Conds:RS_28._col0=RS_29._col0(Inner),Output:["_col0","_col1","_col2","_col4"]
<-Reducer 10 [SIMPLE_EDGE]
SHUFFLE [RS_29]
PartitionCols:_col0
- Filter Operator [FIL_26] (rows=1 width=105)
+ Filter Operator [FIL_26] (rows=2 width=62)
predicate:_col0 is not null
- Limit [LIM_24] (rows=1 width=105)
+ Limit [LIM_24] (rows=3 width=76)
Number of rows:5
- Select Operator [SEL_23] (rows=1 width=105)
+ Select Operator [SEL_23] (rows=3 width=76)
Output:["_col0","_col1"]
<-Reducer 9 [SIMPLE_EDGE]
SHUFFLE [RS_22]
- Select Operator [SEL_20] (rows=1 width=105)
+ Select Operator [SEL_20] (rows=3 width=76)
Output:["_col0","_col1","_col2","_col3"]
- Group By Operator [GBY_19] (rows=1 width=101)
+ Group By Operator [GBY_19] (rows=3 width=70)
Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2
<-Map 8 [SIMPLE_EDGE]
SHUFFLE [RS_18]
PartitionCols:_col0, _col1, _col2
- Group By Operator [GBY_17] (rows=1 width=101)
+ Group By Operator [GBY_17] (rows=3 width=70)
Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float
- Filter Operator [FIL_58] (rows=4 width=93)
+ Filter Operator [FIL_58] (rows=6 width=77)
predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0)))
TableScan [TS_14] (rows=20 width=83)
default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
<-Reducer 3 [SIMPLE_EDGE]
SHUFFLE [RS_28]
PartitionCols:_col0
- Filter Operator [FIL_12] (rows=1 width=97)
+ Filter Operator [FIL_12] (rows=2 width=54)
predicate:_col0 is not null
- Limit [LIM_10] (rows=1 width=97)
+ Limit [LIM_10] (rows=3 width=68)
Number of rows:5
- Select Operator [SEL_9] (rows=1 width=97)
+ Select Operator [SEL_9] (rows=3 width=68)
Output:["_col0","_col1","_col2"]
<-Reducer 2 [SIMPLE_EDGE]
SHUFFLE [RS_8]
- Select Operator [SEL_6] (rows=1 width=97)
+ Select Operator [SEL_6] (rows=3 width=68)
Output:["_col0","_col1","_col2"]
- Group By Operator [GBY_5] (rows=1 width=101)
+ Group By Operator [GBY_5] (rows=3 width=70)
Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_4]
PartitionCols:_col0, _col1, _col2
- Group By Operator [GBY_3] (rows=1 width=101)
+ Group By Operator [GBY_3] (rows=3 width=70)
Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float
- Filter Operator [FIL_56] (rows=4 width=93)
+ Filter Operator [FIL_56] (rows=6 width=77)
predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0)))
TableScan [TS_0] (rows=20 width=83)
default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
@@ -1575,16 +1575,16 @@ Stage-0
Stage-1
Reducer 2
File Output Operator [FS_12]
- Select Operator [SEL_11] (rows=6 width=4)
+ Select Operator [SEL_11] (rows=11 width=4)
Output:["_col0"]
- Merge Join Operator [MERGEJOIN_17] (rows=6 width=4)
+ Merge Join Operator [MERGEJOIN_17] (rows=11 width=4)
Conds:RS_8._col0=RS_9._col0(Left Semi),Output:["_col1"]
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_8]
PartitionCols:_col0
- Select Operator [SEL_2] (rows=5 width=74)
+ Select Operator [SEL_2] (rows=9 width=82)
Output:["_col0","_col1"]
- Filter Operator [FIL_15] (rows=5 width=74)
+ Filter Operator [FIL_15] (rows=9 width=82)
predicate:((((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0))) and key is not null)
TableScan [TS_0] (rows=20 width=83)
default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
@@ -1615,27 +1615,27 @@ Stage-0
Stage-1
Reducer 2
File Output Operator [FS_18]
- Select Operator [SEL_17] (rows=12 width=93)
+ Select Operator [SEL_17] (rows=16 width=93)
Output:["_col0","_col1","_col2"]
- Merge Join Operator [MERGEJOIN_28] (rows=12 width=93)
+ Merge Join Operator [MERGEJOIN_28] (rows=16 width=93)
Conds:RS_13._col0=RS_14._col0(Left Semi),RS_13._col0=RS_15._col0(Left Semi),Output:["_col0","_col1","_col2"]
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_13]
PartitionCols:_col0
- Select Operator [SEL_2] (rows=5 width=74)
+ Select Operator [SEL_2] (rows=9 width=82)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_25] (rows=5 width=74)
+ Filter Operator [FIL_25] (rows=9 width=82)
predicate:((((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0))) and key is not null)
TableScan [TS_0] (rows=20 width=83)
default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
<-Map 3 [SIMPLE_EDGE]
SHUFFLE [RS_14]
PartitionCols:_col0
- Group By Operator [GBY_10] (rows=2 width=85)
+ Group By Operator [GBY_10] (rows=3 width=85)
Output:["_col0"],keys:_col0
- Select Operator [SEL_5] (rows=5 width=68)
+ Select Operator [SEL_5] (rows=9 width=75)
Output:["_col0"]
- Filter Operator [FIL_26] (rows=5 width=74)
+ Filter Operator [FIL_26] (rows=9 width=82)
predicate:((((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0))) and key is not null)
TableScan [TS_3] (rows=20 width=83)
default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
[37/50] [abbrv] hive git commit: HIVE-10729: Query failed when select
complex columns from joinned table (tez map join only) (Matt McCline,
reviewed by Sergey Shelukhin)
Posted by jd...@apache.org.
HIVE-10729: Query failed when select complex columns from joinned table (tez map join only) (Matt McCline, reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ff10f033
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ff10f033
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ff10f033
Branch: refs/heads/llap
Commit: ff10f03371f5ff54d34a28938c5d6e69940113ea
Parents: 44ab455
Author: Matt McCline <mm...@hortonworks.com>
Authored: Tue Mar 29 01:52:48 2016 -0700
Committer: Matt McCline <mm...@hortonworks.com>
Committed: Tue Mar 29 01:54:51 2016 -0700
----------------------------------------------------------------------
.../test/resources/testconfiguration.properties | 1 +
.../ql/exec/vector/VectorizationContext.java | 14 +-
.../mapjoin/VectorMapJoinCommonOperator.java | 2 +-
.../hive/ql/optimizer/physical/Vectorizer.java | 7 +
.../ql/optimizer/physical/TestVectorizer.java | 5 +
.../clientpositive/vector_complex_join.q | 29 +++
.../tez/vector_complex_join.q.out | 227 +++++++++++++++++++
.../clientpositive/vector_complex_join.q.out | 225 ++++++++++++++++++
8 files changed, 502 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/ff10f033/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 0672e0e..ed26dea 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -258,6 +258,7 @@ minitez.query.files.shared=acid_globallimit.q,\
vector_coalesce.q,\
vector_coalesce_2.q,\
vector_complex_all.q,\
+ vector_complex_join.q,\
vector_count_distinct.q,\
vector_data_types.q,\
vector_date_1.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/ff10f033/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
index 0552f9d..1eb960d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
@@ -342,7 +342,7 @@ public class VectorizationContext {
private final Set<Integer> usedOutputColumns = new HashSet<Integer>();
- int allocateOutputColumn(String hiveTypeName) {
+ int allocateOutputColumn(String hiveTypeName) throws HiveException {
if (initialOutputCol < 0) {
// This is a test
return 0;
@@ -403,7 +403,7 @@ public class VectorizationContext {
}
}
- public int allocateScratchColumn(String hiveTypeName) {
+ public int allocateScratchColumn(String hiveTypeName) throws HiveException {
return ocm.allocateOutputColumn(hiveTypeName);
}
@@ -2243,7 +2243,7 @@ public class VectorizationContext {
}
}
- static String getNormalizedName(String hiveTypeName) {
+ static String getNormalizedName(String hiveTypeName) throws HiveException {
VectorExpressionDescriptor.ArgumentType argType = VectorExpressionDescriptor.ArgumentType.fromHiveTypeName(hiveTypeName);
switch (argType) {
case INT_FAMILY:
@@ -2269,11 +2269,11 @@ public class VectorizationContext {
case INTERVAL_DAY_TIME:
return hiveTypeName;
default:
- return "None";
+ throw new HiveException("Unexpected hive type name " + hiveTypeName);
}
}
- static String getUndecoratedName(String hiveTypeName) {
+ static String getUndecoratedName(String hiveTypeName) throws HiveException {
VectorExpressionDescriptor.ArgumentType argType = VectorExpressionDescriptor.ArgumentType.fromHiveTypeName(hiveTypeName);
switch (argType) {
case INT_FAMILY:
@@ -2296,7 +2296,7 @@ public class VectorizationContext {
case INTERVAL_DAY_TIME:
return hiveTypeName;
default:
- return "None";
+ throw new HiveException("Unexpected hive type name " + hiveTypeName);
}
}
@@ -2511,7 +2511,7 @@ public class VectorizationContext {
}
sb.append("sorted projectionColumnMap ").append(sortedColumnMap).append(", ");
- sb.append("scratchColumnTypeNames ").append(getScratchColumnTypeNames().toString());
+ sb.append("scratchColumnTypeNames ").append(Arrays.toString(getScratchColumnTypeNames()));
return sb.toString();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ff10f033/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java
index e26e31b..8ad7ca4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java
@@ -255,7 +255,7 @@ public abstract class VectorMapJoinCommonOperator extends MapJoinOperator implem
determineCommonInfo(isOuterJoin);
}
- protected void determineCommonInfo(boolean isOuter) {
+ protected void determineCommonInfo(boolean isOuter) throws HiveException {
bigTableRetainedMapping = new VectorColumnOutputMapping("Big Table Retained Mapping");
http://git-wip-us.apache.org/repos/asf/hive/blob/ff10f033/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
index f674ece..d806b97 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
@@ -1362,6 +1362,13 @@ public class Vectorizer implements PhysicalPlanResolver {
LOG.info("Cannot vectorize map work value expression");
return false;
}
+ Byte[] order = desc.getTagOrder();
+ Byte posSingleVectorMapJoinSmallTable = (order[0] == posBigTable ? order[1] : order[0]);
+ List<ExprNodeDesc> smallTableExprs = desc.getExprs().get(posSingleVectorMapJoinSmallTable);
+ if (!validateExprNodeDesc(smallTableExprs)) {
+ LOG.info("Cannot vectorize map work small table expression");
+ return false;
+ }
return true;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ff10f033/ql/src/test/org/apache/hadoop/hive/ql/optimizer/physical/TestVectorizer.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/optimizer/physical/TestVectorizer.java b/ql/src/test/org/apache/hadoop/hive/ql/optimizer/physical/TestVectorizer.java
index 5628959..9d4ca76 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/optimizer/physical/TestVectorizer.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/optimizer/physical/TestVectorizer.java
@@ -158,8 +158,13 @@ public class TestVectorizer {
expr.add(new ExprNodeColumnDesc(Integer.class, "col1", "T", false));
Map<Byte, List<ExprNodeDesc>> keyMap = new HashMap<Byte, List<ExprNodeDesc>>();
keyMap.put((byte)0, expr);
+ List<ExprNodeDesc> smallTableExpr = new ArrayList<ExprNodeDesc>();
+ smallTableExpr.add(new ExprNodeColumnDesc(Integer.class, "col2", "T1", false));
+ keyMap.put((byte)1, smallTableExpr);
mjdesc.setKeys(keyMap);
mjdesc.setExprs(keyMap);
+ Byte[] order = new Byte[] {(byte) 0, (byte) 1};
+ mjdesc.setTagOrder(order);
//Set filter expression
GenericUDFOPEqual udf = new GenericUDFOPEqual();
http://git-wip-us.apache.org/repos/asf/hive/blob/ff10f033/ql/src/test/queries/clientpositive/vector_complex_join.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vector_complex_join.q b/ql/src/test/queries/clientpositive/vector_complex_join.q
new file mode 100644
index 0000000..30f38b1
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/vector_complex_join.q
@@ -0,0 +1,29 @@
+set hive.cli.print.header=true;
+set hive.explain.user=false;
+set hive.fetch.task.conversion=none;
+SET hive.vectorized.execution.enabled=true;
+SET hive.auto.convert.join=true;
+set hive.fetch.task.conversion=none;
+
+-- From HIVE-10729. Not expected to vectorize this query.
+--
+CREATE TABLE test (a INT, b MAP<INT, STRING>) STORED AS ORC;
+INSERT OVERWRITE TABLE test SELECT 199408978, MAP(1, "val_1", 2, "val_2") FROM src LIMIT 1;
+
+explain
+select * from alltypesorc join test where alltypesorc.cint=test.a;
+
+select * from alltypesorc join test where alltypesorc.cint=test.a;
+
+
+
+CREATE TABLE test2a (a ARRAY<INT>) STORED AS ORC;
+INSERT OVERWRITE TABLE test2a SELECT ARRAY(1, 2) FROM src LIMIT 1;
+
+CREATE TABLE test2b (a INT) STORED AS ORC;
+INSERT OVERWRITE TABLE test2b VALUES (2), (3), (4);
+
+explain
+select * from test2b join test2a on test2b.a = test2a.a[1];
+
+select * from test2b join test2a on test2b.a = test2a.a[1];
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/ff10f033/ql/src/test/results/clientpositive/tez/vector_complex_join.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_complex_join.q.out b/ql/src/test/results/clientpositive/tez/vector_complex_join.q.out
new file mode 100644
index 0000000..dc988ef
--- /dev/null
+++ b/ql/src/test/results/clientpositive/tez/vector_complex_join.q.out
@@ -0,0 +1,227 @@
+PREHOOK: query: -- From HIVE-10729. Not expected to vectorize this query.
+--
+CREATE TABLE test (a INT, b MAP<INT, STRING>) STORED AS ORC
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@test
+POSTHOOK: query: -- From HIVE-10729. Not expected to vectorize this query.
+--
+CREATE TABLE test (a INT, b MAP<INT, STRING>) STORED AS ORC
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@test
+PREHOOK: query: INSERT OVERWRITE TABLE test SELECT 199408978, MAP(1, "val_1", 2, "val_2") FROM src LIMIT 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@test
+POSTHOOK: query: INSERT OVERWRITE TABLE test SELECT 199408978, MAP(1, "val_1", 2, "val_2") FROM src LIMIT 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@test
+POSTHOOK: Lineage: test.a SIMPLE []
+POSTHOOK: Lineage: test.b EXPRESSION []
+c0 c1
+PREHOOK: query: explain
+select * from alltypesorc join test where alltypesorc.cint=test.a
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select * from alltypesorc join test where alltypesorc.cint=test.a
+POSTHOOK: type: QUERY
+Explain
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Map 1 <- Map 2 (BROADCAST_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypesorc
+ Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: cint is not null (type: boolean)
+ Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11
+ Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col2 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13
+ input vertices:
+ 1 Map 2
+ Statistics: Num rows: 13516 Data size: 2906160 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 13516 Data size: 2906160 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Map 2
+ Map Operator Tree:
+ TableScan
+ alias: test
+ Statistics: Num rows: 1 Data size: 190 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: a is not null (type: boolean)
+ Statistics: Num rows: 1 Data size: 190 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: a (type: int), b (type: map<int,string>)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 190 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 1 Data size: 190 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: map<int,string>)
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select * from alltypesorc join test where alltypesorc.cint=test.a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+PREHOOK: Input: default@test
+#### A masked pattern was here ####
+POSTHOOK: query: select * from alltypesorc join test where alltypesorc.cint=test.a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+POSTHOOK: Input: default@test
+#### A masked pattern was here ####
+alltypesorc.ctinyint alltypesorc.csmallint alltypesorc.cint alltypesorc.cbigint alltypesorc.cfloat alltypesorc.cdouble alltypesorc.cstring1 alltypesorc.cstring2 alltypesorc.ctimestamp1 alltypesorc.ctimestamp2 alltypesorc.cboolean1 alltypesorc.cboolean2 test.a test.b
+-51 NULL 199408978 -1800989684 -51.0 NULL 34N4EY63M1GFWuW0boW P4PL5h1eXR4mMLr2 1969-12-31 16:00:08.451 NULL false true 199408978 {1:"val_1",2:"val_2"}
+PREHOOK: query: CREATE TABLE test2a (a ARRAY<INT>) STORED AS ORC
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@test2a
+POSTHOOK: query: CREATE TABLE test2a (a ARRAY<INT>) STORED AS ORC
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@test2a
+PREHOOK: query: INSERT OVERWRITE TABLE test2a SELECT ARRAY(1, 2) FROM src LIMIT 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@test2a
+POSTHOOK: query: INSERT OVERWRITE TABLE test2a SELECT ARRAY(1, 2) FROM src LIMIT 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@test2a
+POSTHOOK: Lineage: test2a.a EXPRESSION []
+c0
+PREHOOK: query: CREATE TABLE test2b (a INT) STORED AS ORC
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@test2b
+POSTHOOK: query: CREATE TABLE test2b (a INT) STORED AS ORC
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@test2b
+PREHOOK: query: INSERT OVERWRITE TABLE test2b VALUES (2), (3), (4)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@values__tmp__table__1
+PREHOOK: Output: default@test2b
+POSTHOOK: query: INSERT OVERWRITE TABLE test2b VALUES (2), (3), (4)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@values__tmp__table__1
+POSTHOOK: Output: default@test2b
+POSTHOOK: Lineage: test2b.a EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+_col0
+PREHOOK: query: explain
+select * from test2b join test2a on test2b.a = test2a.a[1]
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select * from test2b join test2a on test2b.a = test2a.a[1]
+POSTHOOK: type: QUERY
+Explain
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Map 1 <- Map 2 (BROADCAST_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: test2b
+ Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: a is not null (type: boolean)
+ Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 a (type: int)
+ 1 a[1] (type: int)
+ outputColumnNames: _col0, _col4
+ input vertices:
+ 1 Map 2
+ Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
+ Select Operator
+ expressions: _col0 (type: int), _col4 (type: array<int>)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Map 2
+ Map Operator Tree:
+ TableScan
+ alias: test2a
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: a[1] is not null (type: boolean)
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: a[1] (type: int)
+ sort order: +
+ Map-reduce partition columns: a[1] (type: int)
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: a (type: array<int>)
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select * from test2b join test2a on test2b.a = test2a.a[1]
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test2a
+PREHOOK: Input: default@test2b
+#### A masked pattern was here ####
+POSTHOOK: query: select * from test2b join test2a on test2b.a = test2a.a[1]
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test2a
+POSTHOOK: Input: default@test2b
+#### A masked pattern was here ####
+test2b.a test2a.a
+2 [1,2]
http://git-wip-us.apache.org/repos/asf/hive/blob/ff10f033/ql/src/test/results/clientpositive/vector_complex_join.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_complex_join.q.out b/ql/src/test/results/clientpositive/vector_complex_join.q.out
new file mode 100644
index 0000000..002cdeb
--- /dev/null
+++ b/ql/src/test/results/clientpositive/vector_complex_join.q.out
@@ -0,0 +1,225 @@
+PREHOOK: query: -- From HIVE-10729. Not expected to vectorize this query.
+--
+CREATE TABLE test (a INT, b MAP<INT, STRING>) STORED AS ORC
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@test
+POSTHOOK: query: -- From HIVE-10729. Not expected to vectorize this query.
+--
+CREATE TABLE test (a INT, b MAP<INT, STRING>) STORED AS ORC
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@test
+PREHOOK: query: INSERT OVERWRITE TABLE test SELECT 199408978, MAP(1, "val_1", 2, "val_2") FROM src LIMIT 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@test
+POSTHOOK: query: INSERT OVERWRITE TABLE test SELECT 199408978, MAP(1, "val_1", 2, "val_2") FROM src LIMIT 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@test
+POSTHOOK: Lineage: test.a SIMPLE []
+POSTHOOK: Lineage: test.b EXPRESSION []
+c0 c1
+PREHOOK: query: explain
+select * from alltypesorc join test where alltypesorc.cint=test.a
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select * from alltypesorc join test where alltypesorc.cint=test.a
+POSTHOOK: type: QUERY
+Explain
+STAGE DEPENDENCIES:
+ Stage-4 is a root stage
+ Stage-3 depends on stages: Stage-4
+ Stage-0 depends on stages: Stage-3
+
+STAGE PLANS:
+ Stage: Stage-4
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ $hdt$_1:test
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ $hdt$_1:test
+ TableScan
+ alias: test
+ Statistics: Num rows: 1 Data size: 190 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: a is not null (type: boolean)
+ Statistics: Num rows: 1 Data size: 190 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: a (type: int), b (type: map<int,string>)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 190 Basic stats: COMPLETE Column stats: NONE
+ HashTable Sink Operator
+ keys:
+ 0 _col2 (type: int)
+ 1 _col0 (type: int)
+
+ Stage: Stage-3
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: alltypesorc
+ Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: cint is not null (type: boolean)
+ Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11
+ Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col2 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13
+ Statistics: Num rows: 13516 Data size: 2906160 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 13516 Data size: 2906160 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select * from alltypesorc join test where alltypesorc.cint=test.a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+PREHOOK: Input: default@test
+#### A masked pattern was here ####
+POSTHOOK: query: select * from alltypesorc join test where alltypesorc.cint=test.a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+POSTHOOK: Input: default@test
+#### A masked pattern was here ####
+alltypesorc.ctinyint alltypesorc.csmallint alltypesorc.cint alltypesorc.cbigint alltypesorc.cfloat alltypesorc.cdouble alltypesorc.cstring1 alltypesorc.cstring2 alltypesorc.ctimestamp1 alltypesorc.ctimestamp2 alltypesorc.cboolean1 alltypesorc.cboolean2 test.a test.b
+-51 NULL 199408978 -1800989684 -51.0 NULL 34N4EY63M1GFWuW0boW P4PL5h1eXR4mMLr2 1969-12-31 16:00:08.451 NULL false true 199408978 {1:"val_1",2:"val_2"}
+PREHOOK: query: CREATE TABLE test2a (a ARRAY<INT>) STORED AS ORC
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@test2a
+POSTHOOK: query: CREATE TABLE test2a (a ARRAY<INT>) STORED AS ORC
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@test2a
+PREHOOK: query: INSERT OVERWRITE TABLE test2a SELECT ARRAY(1, 2) FROM src LIMIT 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@test2a
+POSTHOOK: query: INSERT OVERWRITE TABLE test2a SELECT ARRAY(1, 2) FROM src LIMIT 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@test2a
+POSTHOOK: Lineage: test2a.a EXPRESSION []
+c0
+PREHOOK: query: CREATE TABLE test2b (a INT) STORED AS ORC
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@test2b
+POSTHOOK: query: CREATE TABLE test2b (a INT) STORED AS ORC
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@test2b
+PREHOOK: query: INSERT OVERWRITE TABLE test2b VALUES (2), (3), (4)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@values__tmp__table__1
+PREHOOK: Output: default@test2b
+POSTHOOK: query: INSERT OVERWRITE TABLE test2b VALUES (2), (3), (4)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@values__tmp__table__1
+POSTHOOK: Output: default@test2b
+POSTHOOK: Lineage: test2b.a EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+_col0
+PREHOOK: query: explain
+select * from test2b join test2a on test2b.a = test2a.a[1]
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select * from test2b join test2a on test2b.a = test2a.a[1]
+POSTHOOK: type: QUERY
+Explain
+STAGE DEPENDENCIES:
+ Stage-4 is a root stage
+ Stage-3 depends on stages: Stage-4
+ Stage-0 depends on stages: Stage-3
+
+STAGE PLANS:
+ Stage: Stage-4
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ test2b
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ test2b
+ TableScan
+ alias: test2b
+ Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: a is not null (type: boolean)
+ Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ HashTable Sink Operator
+ keys:
+ 0 a (type: int)
+ 1 a[1] (type: int)
+
+ Stage: Stage-3
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: test2a
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: a[1] is not null (type: boolean)
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 a (type: int)
+ 1 a[1] (type: int)
+ outputColumnNames: _col0, _col4
+ Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: int), _col4 (type: array<int>)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select * from test2b join test2a on test2b.a = test2a.a[1]
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test2a
+PREHOOK: Input: default@test2b
+#### A masked pattern was here ####
+POSTHOOK: query: select * from test2b join test2a on test2b.a = test2a.a[1]
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test2a
+POSTHOOK: Input: default@test2b
+#### A masked pattern was here ####
+test2b.a test2a.a
+2 [1,2]
[09/50] [abbrv] hive git commit: HIVE-13262: LLAP: Remove log levels
from DebugUtils (Prasanth Jayachandran reviewed by Sergey Shelukhin)
Posted by jd...@apache.org.
HIVE-13262: LLAP: Remove log levels from DebugUtils (Prasanth Jayachandran reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3b6b56d7
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3b6b56d7
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3b6b56d7
Branch: refs/heads/llap
Commit: 3b6b56d7000ee1d80c0f191611968d4249f311d7
Parents: dfba1fb
Author: Prasanth Jayachandran <pr...@apache.org>
Authored: Thu Mar 24 20:49:30 2016 -0500
Committer: Prasanth Jayachandran <pr...@apache.org>
Committed: Thu Mar 24 20:49:30 2016 -0500
----------------------------------------------------------------------
.../llap/IncrementalObjectSizeEstimator.java | 54 ++---
.../hadoop/hive/llap/cache/LlapDataBuffer.java | 12 +-
.../hive/llap/cache/LowLevelCacheImpl.java | 35 ++-
.../llap/cache/LowLevelCacheMemoryManager.java | 6 +-
.../llap/cache/LowLevelFifoCachePolicy.java | 4 +-
.../llap/cache/LowLevelLrfuCachePolicy.java | 14 +-
.../hive/llap/cache/SimpleBufferManager.java | 8 +-
.../hive/llap/io/api/impl/LlapInputFormat.java | 32 +--
.../hive/llap/io/api/impl/LlapIoImpl.java | 21 +-
.../llap/io/decode/OrcColumnVectorProducer.java | 4 +-
.../llap/io/encoded/OrcEncodedDataReader.java | 95 +++-----
.../hadoop/hive/llap/old/BufferInProgress.java | 82 -------
.../apache/hadoop/hive/llap/old/BufferPool.java | 225 ------------------
.../hadoop/hive/llap/old/CachePolicy.java | 34 ---
.../apache/hadoop/hive/llap/old/ChunkPool.java | 237 -------------------
.../resources/llap-daemon-log4j2.properties | 14 +-
.../org/apache/hadoop/hive/llap/DebugUtils.java | 43 ----
.../org/apache/hadoop/hive/llap/LogLevels.java | 53 -----
.../ql/exec/vector/VectorGroupByOperator.java | 2 +-
.../hive/ql/io/orc/encoded/EncodedReader.java | 2 +-
.../ql/io/orc/encoded/EncodedReaderImpl.java | 131 +++++-----
21 files changed, 192 insertions(+), 916 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/3b6b56d7/llap-server/src/java/org/apache/hadoop/hive/llap/IncrementalObjectSizeEstimator.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/IncrementalObjectSizeEstimator.java b/llap-server/src/java/org/apache/hadoop/hive/llap/IncrementalObjectSizeEstimator.java
index 7d68294..3efbcc2 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/IncrementalObjectSizeEstimator.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/IncrementalObjectSizeEstimator.java
@@ -186,9 +186,7 @@ public class IncrementalObjectSizeEstimator {
fieldCol = (Collection<?>)fieldObj;
if (fieldCol.size() == 0) {
fieldCol = null;
- if (DebugUtils.isTraceEnabled()) {
- LlapIoImpl.LOG.info("Empty collection " + field);
- }
+ LlapIoImpl.LOG.trace("Empty collection {}", field);
}
}
if (fieldCol != null) {
@@ -219,9 +217,7 @@ public class IncrementalObjectSizeEstimator {
fieldCol = (Map<?, ?>)fieldObj;
if (fieldCol.size() == 0) {
fieldCol = null;
- if (DebugUtils.isTraceEnabled()) {
- LlapIoImpl.LOG.info("Empty map " + field);
- }
+ LlapIoImpl.LOG.trace("Empty map {}", field);
}
}
if (fieldCol != null) {
@@ -257,15 +253,11 @@ public class IncrementalObjectSizeEstimator {
return new Class<?>[] { (Class<?>)types[0], (Class<?>)types[1] };
} else {
// TODO: we could try to get the declaring object and infer argument... stupid Java.
- if (DebugUtils.isTraceEnabled()) {
- LlapIoImpl.LOG.info("Cannot determine map type: " + field);
- }
+ LlapIoImpl.LOG.trace("Cannot determine map type: {}", field);
}
} else {
// TODO: we could try to get superclass or generic interfaces.
- if (DebugUtils.isTraceEnabled()) {
- LlapIoImpl.LOG.info("Non-parametrized map type: " + field);
- }
+ LlapIoImpl.LOG.trace("Non-parametrized map type: {}", field);
}
return null;
}
@@ -279,15 +271,11 @@ public class IncrementalObjectSizeEstimator {
return (Class<?>)type;
} else {
// TODO: we could try to get the declaring object and infer argument... stupid Java.
- if (DebugUtils.isTraceEnabled()) {
- LlapIoImpl.LOG.info("Cannot determine collection type: " + field);
- }
+ LlapIoImpl.LOG.trace("Cannot determine collection type: {}", field);
}
} else {
// TODO: we could try to get superclass or generic interfaces.
- if (DebugUtils.isTraceEnabled()) {
- LlapIoImpl.LOG.info("Non-parametrized collection type: " + field);
- }
+ LlapIoImpl.LOG.trace("Non-parametrized collection type: {}", field);
}
return null;
}
@@ -297,11 +285,7 @@ public class IncrementalObjectSizeEstimator {
Field field, Object fieldObj) {
if (fieldObj == null) return;
int arrayLen = Array.getLength(fieldObj);
- if (arrayLen == 0) {
- if (DebugUtils.isTraceEnabled()) {
- LlapIoImpl.LOG.info("Empty array " + field);
- }
- }
+ LlapIoImpl.LOG.trace("Empty array {}", field);
for (int i = 0; i < arrayLen; ++i) {
Object element = Array.get(fieldObj, i);
if (element != null) {
@@ -416,10 +400,8 @@ public class IncrementalObjectSizeEstimator {
ObjectEstimator collEstimator = parent.get(fieldObj.getClass());
if (collEstimator == null) {
// We have no estimator for this type... assume low overhead and hope for the best.
- if (DebugUtils.isTraceEnabled()) {
- LlapIoImpl.LOG.info("Approximate estimation for collection "
- + fieldObj.getClass().getName() + " from " + e.field);
- }
+ LlapIoImpl.LOG.trace("Approximate estimation for collection {} from {}", e.field,
+ fieldObj.getClass().getName());
referencedSize += memoryModel.object();
referencedSize += estimateCollectionElements(parent, c, e.field, uniqueObjects);
referencedSize += memoryModel.array() + c.size() * memoryModel.ref();
@@ -429,10 +411,8 @@ public class IncrementalObjectSizeEstimator {
referencedSize += ((CollectionEstimator)collEstimator).estimateOverhead(c.size());
} else {
// We decided to treat this collection as regular object.
- if (DebugUtils.isTraceEnabled()) {
- LlapIoImpl.LOG.info("Verbose estimation for collection "
- + fieldObj.getClass().getName() + " from " + e.field);
- }
+ LlapIoImpl.LOG.trace("Verbose estimation for collection {} from {}",
+ fieldObj.getClass().getName(), e.field);
referencedSize += collEstimator.estimate(c, parent, uniqueObjects);
}
break;
@@ -442,10 +422,8 @@ public class IncrementalObjectSizeEstimator {
ObjectEstimator collEstimator = parent.get(fieldObj.getClass());
if (collEstimator == null) {
// We have no estimator for this type... assume low overhead and hope for the best.
- if (DebugUtils.isTraceEnabled()) {
- LlapIoImpl.LOG.info("Approximate estimation for map "
- + fieldObj.getClass().getName() + " from " + e.field);
- }
+ LlapIoImpl.LOG.trace("Approximate estimation for map {} from {}",
+ fieldObj.getClass().getName(), e.field);
referencedSize += memoryModel.object();
referencedSize += estimateMapElements(parent, m, e.field, uniqueObjects);
referencedSize += memoryModel.array() + m.size()
@@ -456,10 +434,8 @@ public class IncrementalObjectSizeEstimator {
referencedSize += ((CollectionEstimator)collEstimator).estimateOverhead(m.size());
} else {
// We decided to treat this map as regular object.
- if (DebugUtils.isTraceEnabled()) {
- LlapIoImpl.LOG.info("Verbose estimation for map "
- + fieldObj.getClass().getName() + " from " + e.field);
- }
+ LlapIoImpl.LOG.trace("Verbose estimation for map {} from {}",
+ fieldObj.getClass().getName(), e.field);
referencedSize += collEstimator.estimate(m, parent, uniqueObjects);
}
break;
http://git-wip-us.apache.org/repos/asf/hive/blob/3b6b56d7/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LlapDataBuffer.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LlapDataBuffer.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LlapDataBuffer.java
index 840aeab..d1a961c 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LlapDataBuffer.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LlapDataBuffer.java
@@ -83,8 +83,8 @@ public final class LlapDataBuffer extends LlapCacheableBuffer implements MemoryB
newRefCount = oldRefCount + 1;
if (refCount.compareAndSet(oldRefCount, newRefCount)) break;
}
- if (DebugUtils.isTraceLockingEnabled()) {
- LlapIoImpl.LOG.info("Locked " + this + "; new ref count " + newRefCount);
+ if (LlapIoImpl.LOCKING_LOGGER.isTraceEnabled()) {
+ LlapIoImpl.LOCKING_LOGGER.trace("Locked {}; new ref count {}", this, newRefCount);
}
return newRefCount;
}
@@ -109,8 +109,8 @@ public final class LlapDataBuffer extends LlapCacheableBuffer implements MemoryB
int decRef() {
int newRefCount = refCount.decrementAndGet();
- if (DebugUtils.isTraceLockingEnabled()) {
- LlapIoImpl.LOG.info("Unlocked " + this + "; refcount " + newRefCount);
+ if (LlapIoImpl.LOCKING_LOGGER.isTraceEnabled()) {
+ LlapIoImpl.LOCKING_LOGGER.trace("Unlocked {}; refcount {}", this, newRefCount);
}
if (newRefCount < 0) {
throw new AssertionError("Unexpected refCount " + newRefCount + ": " + this);
@@ -128,8 +128,8 @@ public final class LlapDataBuffer extends LlapCacheableBuffer implements MemoryB
if (value != 0) return false;
if (refCount.compareAndSet(value, EVICTED_REFCOUNT)) break;
}
- if (DebugUtils.isTraceLockingEnabled()) {
- LlapIoImpl.LOG.info("Invalidated " + this + " due to eviction");
+ if (LlapIoImpl.LOCKING_LOGGER.isTraceEnabled()) {
+ LlapIoImpl.LOCKING_LOGGER.trace("Invalidated {} due to eviction", this);
}
return true;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/3b6b56d7/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheImpl.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheImpl.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheImpl.java
index a60fed3..038c3ed 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheImpl.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheImpl.java
@@ -58,9 +58,8 @@ public class LowLevelCacheImpl implements LowLevelCache, BufferUsageManager, Lla
@VisibleForTesting
LowLevelCacheImpl(LlapDaemonCacheMetrics metrics, LowLevelCachePolicy cachePolicy,
EvictionAwareAllocator allocator, boolean doAssumeGranularBlocks, long cleanupInterval) {
- if (LlapIoImpl.LOGL.isInfoEnabled()) {
- LlapIoImpl.LOG.info("Low level cache; cleanup interval " + cleanupInterval + "sec");
- }
+
+ LlapIoImpl.LOG.info("Low level cache; cleanup interval {} sec", cleanupInterval);
this.cachePolicy = cachePolicy;
this.allocator = allocator;
this.cleanupInterval = cleanupInterval;
@@ -148,8 +147,8 @@ public class LowLevelCacheImpl implements LowLevelCache, BufferUsageManager, Lla
LlapDataBuffer buffer = e.getValue();
long requestedLength = currentNotCached.getLength();
// Lock the buffer, validate it and add to results.
- if (DebugUtils.isTraceLockingEnabled()) {
- LlapIoImpl.LOG.info("Locking " + buffer + " during get");
+ if (LlapIoImpl.LOCKING_LOGGER.isTraceEnabled()) {
+ LlapIoImpl.LOCKING_LOGGER.trace("Locking {} during get", buffer);
}
if (!lockBuffer(buffer, true)) {
@@ -183,7 +182,6 @@ public class LowLevelCacheImpl implements LowLevelCache, BufferUsageManager, Lla
* Adds cached buffer to buffer list.
* @param currentNotCached Pointer to the list node where we are inserting.
* @param currentCached The cached buffer found for this node, to insert.
- * @param resultObj
* @return The new currentNotCached pointer, following the cached buffer insertion.
*/
private DiskRangeList addCachedBufferToIter(
@@ -240,8 +238,8 @@ public class LowLevelCacheImpl implements LowLevelCache, BufferUsageManager, Lla
try {
for (int i = 0; i < ranges.length; ++i) {
LlapDataBuffer buffer = (LlapDataBuffer)buffers[i];
- if (DebugUtils.isTraceLockingEnabled()) {
- LlapIoImpl.LOG.info("Locking " + buffer + " at put time");
+ if (LlapIoImpl.LOCKING_LOGGER.isTraceEnabled()) {
+ LlapIoImpl.LOCKING_LOGGER.trace("Locking {} at put time", buffer);
}
boolean canLock = lockBuffer(buffer, false);
assert canLock;
@@ -258,13 +256,13 @@ public class LowLevelCacheImpl implements LowLevelCache, BufferUsageManager, Lla
}
break;
}
- if (DebugUtils.isTraceCachingEnabled()) {
- LlapIoImpl.LOG.info("Trying to cache when the chunk is already cached for "
- + fileKey + "@" + offset + " (base " + baseOffset + "); old " + oldVal
- + ", new " + buffer);
+ if (LlapIoImpl.CACHE_LOGGER.isTraceEnabled()) {
+ LlapIoImpl.CACHE_LOGGER.trace("Trying to cache when the chunk is already cached for" +
+ " {}@{} (base {}); old {}, new {}", fileKey, offset, baseOffset, oldVal, buffer);
}
- if (DebugUtils.isTraceLockingEnabled()) {
- LlapIoImpl.LOG.info("Locking " + oldVal + " due to cache collision");
+
+ if (LlapIoImpl.LOCKING_LOGGER.isTraceEnabled()) {
+ LlapIoImpl.LOCKING_LOGGER.trace("Locking {} due to cache collision", oldVal);
}
if (lockBuffer(oldVal, true)) {
// We don't do proper overlap checking because it would cost cycles and we
@@ -275,8 +273,9 @@ public class LowLevelCacheImpl implements LowLevelCache, BufferUsageManager, Lla
+ " (base " + baseOffset + ")");
}
// We found an old, valid block for this key in the cache.
- if (DebugUtils.isTraceLockingEnabled()) {
- LlapIoImpl.LOG.info("Unlocking " + buffer + " due to cache collision with " + oldVal);
+ if (LlapIoImpl.LOCKING_LOGGER.isTraceEnabled()) {
+ LlapIoImpl.LOCKING_LOGGER.trace("Unlocking {} due to cache collision with {}",
+ buffer, oldVal);
}
unlockBuffer(buffer, false);
@@ -353,8 +352,8 @@ public class LowLevelCacheImpl implements LowLevelCache, BufferUsageManager, Lla
if (buffer.declaredCachedLength != LlapDataBuffer.UNKNOWN_CACHED_LENGTH) {
cachePolicy.notifyUnlock(buffer);
} else {
- if (DebugUtils.isTraceCachingEnabled()) {
- LlapIoImpl.LOG.info("Deallocating " + buffer + " that was not cached");
+ if (LlapIoImpl.CACHE_LOGGER.isTraceEnabled()) {
+ LlapIoImpl.CACHE_LOGGER.trace("Deallocating {} that was not cached", buffer);
}
allocator.deallocate(buffer);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/3b6b56d7/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheMemoryManager.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheMemoryManager.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheMemoryManager.java
index 1cfe2bc..4def4a1 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheMemoryManager.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheMemoryManager.java
@@ -50,9 +50,9 @@ public class LowLevelCacheMemoryManager implements MemoryManager {
this.usedMemory = new AtomicLong(0);
this.metrics = metrics;
metrics.setCacheCapacityTotal(maxSize);
- if (LlapIoImpl.LOGL.isInfoEnabled()) {
- LlapIoImpl.LOG.info("Memory manager initialized with max size " + maxSize + " and "
- + ((evictor == null) ? "no " : "") + "ability to evict blocks");
+ if (LlapIoImpl.LOG.isInfoEnabled()) {
+ LlapIoImpl.LOG.info("Memory manager initialized with max size {} and" +
+ " {} ability to evict blocks", maxSize, ((evictor == null) ? "no " : ""));
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/3b6b56d7/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelFifoCachePolicy.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelFifoCachePolicy.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelFifoCachePolicy.java
index 1430eae..0838682 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelFifoCachePolicy.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelFifoCachePolicy.java
@@ -35,9 +35,7 @@ public class LowLevelFifoCachePolicy implements LowLevelCachePolicy {
private LlapOomDebugDump parentDebugDump;
public LowLevelFifoCachePolicy(Configuration conf) {
- if (LlapIoImpl.LOGL.isInfoEnabled()) {
- LlapIoImpl.LOG.info("FIFO cache policy");
- }
+ LlapIoImpl.LOG.info("FIFO cache policy");
buffers = new LinkedList<LlapCacheableBuffer>();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/3b6b56d7/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelLrfuCachePolicy.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelLrfuCachePolicy.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelLrfuCachePolicy.java
index 6f52b86..bbff3cc 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelLrfuCachePolicy.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelLrfuCachePolicy.java
@@ -82,10 +82,8 @@ public class LowLevelLrfuCachePolicy implements LowLevelCachePolicy {
int lrfuThreshold = (int)((Math.log(1 - Math.pow(0.5, lambda)) / Math.log(0.5)) / lambda);
maxHeapSize = Math.min(lrfuThreshold, maxBuffers);
}
- if (LlapIoImpl.LOGL.isInfoEnabled()) {
- LlapIoImpl.LOG.info("LRFU cache policy with min buffer size " + minBufferSize
- + " and lambda " + lambda + " (heap size " + maxHeapSize + ")");
- }
+ LlapIoImpl.LOG.info("LRFU cache policy with min buffer size {} and lambda {} (heap size {})",
+ minBufferSize, lambda, maxHeapSize);
heap = new LlapCacheableBuffer[maxHeapSize];
listHead = listTail = null;
@@ -123,8 +121,8 @@ public class LowLevelLrfuCachePolicy implements LowLevelCachePolicy {
@Override
public void notifyUnlock(LlapCacheableBuffer buffer) {
long time = timer.incrementAndGet();
- if (DebugUtils.isTraceCachingEnabled()) {
- LlapIoImpl.LOG.info("Touching " + buffer + " at " + time);
+ if (LlapIoImpl.CACHE_LOGGER.isTraceEnabled()) {
+ LlapIoImpl.CACHE_LOGGER.trace("Touching {} at {}", buffer, time);
}
synchronized (heap) {
// First, update buffer priority - we have just been using it.
@@ -263,8 +261,8 @@ public class LowLevelLrfuCachePolicy implements LowLevelCachePolicy {
while (true) {
if (heapSize == 0) return null;
LlapCacheableBuffer result = heap[0];
- if (DebugUtils.isTraceCachingEnabled()) {
- LlapIoImpl.LOG.info("Evicting " + result + " at " + time);
+ if (LlapIoImpl.CACHE_LOGGER.isTraceEnabled()) {
+ LlapIoImpl.CACHE_LOGGER.info("Evicting {} at {}", result, time);
}
result.indexInHeap = LlapCacheableBuffer.NOT_IN_CACHE;
--heapSize;
http://git-wip-us.apache.org/repos/asf/hive/blob/3b6b56d7/llap-server/src/java/org/apache/hadoop/hive/llap/cache/SimpleBufferManager.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/SimpleBufferManager.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/SimpleBufferManager.java
index 734a5c0..b188c0e 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/SimpleBufferManager.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/SimpleBufferManager.java
@@ -30,9 +30,7 @@ public class SimpleBufferManager implements BufferUsageManager {
private final LlapDaemonCacheMetrics metrics;
public SimpleBufferManager(Allocator allocator, LlapDaemonCacheMetrics metrics) {
- if (LlapIoImpl.LOGL.isInfoEnabled()) {
- LlapIoImpl.LOG.info("Simple buffer manager");
- }
+ LlapIoImpl.LOG.info("Simple buffer manager");
this.allocator = allocator;
this.metrics = metrics;
}
@@ -46,8 +44,8 @@ public class SimpleBufferManager implements BufferUsageManager {
private void unlockBuffer(LlapDataBuffer buffer) {
if (buffer.decRef() == 0) {
- if (DebugUtils.isTraceCachingEnabled()) {
- LlapIoImpl.LOG.info("Deallocating " + buffer + " that was not cached");
+ if (LlapIoImpl.CACHE_LOGGER.isTraceEnabled()) {
+ LlapIoImpl.CACHE_LOGGER.trace("Deallocating {} that was not cached", buffer);
}
allocator.deallocate(buffer);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/3b6b56d7/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java
index 85cca97..9fb79a5 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java
@@ -264,21 +264,21 @@ public class LlapInputFormat implements InputFormat<NullWritable, VectorizedRowB
}
synchronized (pendingData) {
// We are waiting for next block. Either we will get it, or be told we are done.
- boolean doLogBlocking = DebugUtils.isTraceMttEnabled() && isNothingToReport();
+ boolean doLogBlocking = LlapIoImpl.LOG.isTraceEnabled() && isNothingToReport();
if (doLogBlocking) {
- LlapIoImpl.LOG.info("next will block");
+ LlapIoImpl.LOG.trace("next will block");
}
while (isNothingToReport()) {
pendingData.wait(100);
}
if (doLogBlocking) {
- LlapIoImpl.LOG.info("next is unblocked");
+ LlapIoImpl.LOG.trace("next is unblocked");
}
rethrowErrorIfAny();
lastCvb = pendingData.poll();
}
- if (DebugUtils.isTraceMttEnabled() && lastCvb != null) {
- LlapIoImpl.LOG.info("Processing will receive vector " + lastCvb);
+ if (LlapIoImpl.LOG.isTraceEnabled() && lastCvb != null) {
+ LlapIoImpl.LOG.trace("Processing will receive vector {}", lastCvb);
}
return lastCvb;
}
@@ -304,9 +304,9 @@ public class LlapInputFormat implements InputFormat<NullWritable, VectorizedRowB
@Override
public void close() throws IOException {
- if (DebugUtils.isTraceMttEnabled()) {
- LlapIoImpl.LOG.info("close called; closed " + isClosed + ", done " + isDone
- + ", err " + pendingError + ", pending " + pendingData.size());
+ if (LlapIoImpl.LOG.isTraceEnabled()) {
+ LlapIoImpl.LOG.trace("close called; closed {}, done {}, err {}, pending {}",
+ isClosed, isDone, pendingError, pendingData.size());
}
LlapIoImpl.LOG.info("Llap counters: {}" ,counters); // This is where counters are logged!
feedback.stop();
@@ -323,9 +323,9 @@ public class LlapInputFormat implements InputFormat<NullWritable, VectorizedRowB
@Override
public void setDone() {
- if (DebugUtils.isTraceMttEnabled()) {
- LlapIoImpl.LOG.info("setDone called; closed " + isClosed
- + ", done " + isDone + ", err " + pendingError + ", pending " + pendingData.size());
+ if (LlapIoImpl.LOG.isTraceEnabled()) {
+ LlapIoImpl.LOG.trace("setDone called; closed {}, done {}, err {}, pending {}",
+ isClosed, isDone, pendingError, pendingData.size());
}
synchronized (pendingData) {
isDone = true;
@@ -335,9 +335,9 @@ public class LlapInputFormat implements InputFormat<NullWritable, VectorizedRowB
@Override
public void consumeData(ColumnVectorBatch data) {
- if (DebugUtils.isTraceMttEnabled()) {
- LlapIoImpl.LOG.info("consume called; closed " + isClosed + ", done " + isDone
- + ", err " + pendingError + ", pending " + pendingData.size());
+ if (LlapIoImpl.LOG.isTraceEnabled()) {
+ LlapIoImpl.LOG.trace("consume called; closed {}, done {}, err {}, pending {}",
+ isClosed, isDone, pendingError, pendingData.size());
}
synchronized (pendingData) {
if (isClosed) {
@@ -351,8 +351,8 @@ public class LlapInputFormat implements InputFormat<NullWritable, VectorizedRowB
@Override
public void setError(Throwable t) {
counters.incrCounter(LlapIOCounters.NUM_ERRORS);
- LlapIoImpl.LOG.info("setError called; closed " + isClosed
- + ", done " + isDone + ", err " + pendingError + ", pending " + pendingData.size());
+ LlapIoImpl.LOG.info("setError called; closed {}, done {}, err {}, pending {}",
+ isClosed, isDone, pendingError, pendingData.size());
assert t != null;
synchronized (pendingData) {
pendingError = t;
http://git-wip-us.apache.org/repos/asf/hive/blob/3b6b56d7/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java
index dbee823..36f8dec 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.hive.llap.io.api.impl;
-import org.apache.hadoop.hive.llap.LogLevels;
-
import java.io.IOException;
import java.util.concurrent.Executors;
@@ -58,8 +56,11 @@ import com.google.common.util.concurrent.MoreExecutors;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
public class LlapIoImpl implements LlapIo<VectorizedRowBatch> {
- public static final Logger LOG = LoggerFactory.getLogger(LlapIoImpl.class);
- public static final LogLevels LOGL = new LogLevels(LOG);
+ public static final Logger LOG = LoggerFactory.getLogger("LlapIoImpl");
+ public static final Logger ORC_LOGGER = LoggerFactory.getLogger("LlapIoOrc");
+ public static final Logger CACHE_LOGGER = LoggerFactory.getLogger("LlapIoCache");
+ public static final Logger LOCKING_LOGGER = LoggerFactory.getLogger("LlapIoLocking");
+
private static final String MODE_CACHE = "cache", MODE_ALLOCATOR = "allocator";
private final ColumnVectorProducer cvp;
@@ -73,9 +74,7 @@ public class LlapIoImpl implements LlapIo<VectorizedRowBatch> {
String ioMode = HiveConf.getVar(conf, HiveConf.ConfVars.LLAP_IO_MEMORY_MODE);
boolean useLowLevelCache = LlapIoImpl.MODE_CACHE.equalsIgnoreCase(ioMode),
useAllocOnly = !useLowLevelCache && LlapIoImpl.MODE_ALLOCATOR.equalsIgnoreCase(ioMode);
- if (LOGL.isInfoEnabled()) {
- LOG.info("Initializing LLAP IO in " + ioMode + " mode");
- }
+ LOG.info("Initializing LLAP IO in {} mode", ioMode);
String displayName = "LlapDaemonCacheMetrics-" + MetricsUtils.getHostName();
String sessionId = conf.get("llap.daemon.metrics.sessionid");
@@ -86,8 +85,8 @@ public class LlapIoImpl implements LlapIo<VectorizedRowBatch> {
HiveConf.ConfVars.LLAP_QUEUE_METRICS_PERCENTILE_INTERVALS));
this.queueMetrics = LlapDaemonQueueMetrics.create(displayName, sessionId, intervals);
- LOG.info("Started llap daemon metrics with displayName: " + displayName +
- " sessionId: " + sessionId);
+ LOG.info("Started llap daemon metrics with displayName: {} sessionId: {}", displayName,
+ sessionId);
OrcMetadataCache metadataCache = null;
LowLevelCacheImpl orcCache = null;
@@ -128,9 +127,7 @@ public class LlapIoImpl implements LlapIo<VectorizedRowBatch> {
// TODO: this should depends on input format and be in a map, or something.
this.cvp = new OrcColumnVectorProducer(
metadataCache, orcCache, bufferManager, conf, cacheMetrics, queueMetrics);
- if (LOGL.isInfoEnabled()) {
- LOG.info("LLAP IO initialized");
- }
+ LOG.info("LLAP IO initialized");
registerMXBeans();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/3b6b56d7/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcColumnVectorProducer.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcColumnVectorProducer.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcColumnVectorProducer.java
index 37fc8d0..024c485 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcColumnVectorProducer.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcColumnVectorProducer.java
@@ -49,9 +49,7 @@ public class OrcColumnVectorProducer implements ColumnVectorProducer {
public OrcColumnVectorProducer(OrcMetadataCache metadataCache,
LowLevelCacheImpl lowLevelCache, BufferUsageManager bufferManager,
Configuration conf, LlapDaemonCacheMetrics metrics, LlapDaemonQueueMetrics queueMetrics) {
- if (LlapIoImpl.LOGL.isInfoEnabled()) {
- LlapIoImpl.LOG.info("Initializing ORC column vector producer");
- }
+ LlapIoImpl.LOG.info("Initializing ORC column vector producer");
this.metadataCache = metadataCache;
this.lowLevelCache = lowLevelCache;
http://git-wip-us.apache.org/repos/asf/hive/blob/3b6b56d7/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
index eb251a8..fb0867d 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
@@ -184,9 +184,7 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void>
@Override
public void stop() {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Encoded reader is being stopped");
- }
+ LOG.debug("Encoded reader is being stopped");
isStopped = true;
}
@@ -214,9 +212,7 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void>
protected Void performDataRead() throws IOException {
long startTime = counters.startTimeCounter();
- if (LlapIoImpl.LOGL.isInfoEnabled()) {
- LlapIoImpl.LOG.info("Processing data for " + split.getPath());
- }
+ LlapIoImpl.LOG.info("Processing data for {}", split.getPath());
if (processStop()) {
recordReaderTime(startTime);
return null;
@@ -310,7 +306,7 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void>
// Reader creating updates HDFS counters, don't do it here.
DataWrapperForOrc dw = new DataWrapperForOrc();
stripeReader = orcReader.encodedReader(fileKey, dw, dw, POOL_FACTORY);
- stripeReader.setDebugTracing(DebugUtils.isTraceOrcEnabled());
+ stripeReader.setTracing(LlapIoImpl.ORC_LOGGER.isTraceEnabled());
} catch (Throwable t) {
consumer.setError(t);
recordReaderTime(startTime);
@@ -338,10 +334,8 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void>
if (cols != null && cols.isEmpty()) continue; // No need to read this stripe.
stripe = fileMetadata.getStripes().get(stripeIx);
- if (DebugUtils.isTraceOrcEnabled()) {
- LlapIoImpl.LOG.info("Reading stripe " + stripeIx + ": "
- + stripe.getOffset() + ", " + stripe.getLength());
- }
+ LlapIoImpl.ORC_LOGGER.trace("Reading stripe {}: {}, {}", stripeIx, stripe.getOffset(),
+ stripe.getLength());
colRgs = readState[stripeIxMod];
// We assume that NO_RGS value is only set from SARG filter and for all columns;
// intermediate changes for individual columns will unset values in the array.
@@ -379,18 +373,18 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void>
counters.incrTimeCounter(LlapIOCounters.HDFS_TIME_NS, startTimeHdfs);
if (hasFileId && metadataCache != null) {
stripeMetadata = metadataCache.putStripeMetadata(stripeMetadata);
- if (DebugUtils.isTraceOrcEnabled()) {
- LlapIoImpl.LOG.info("Caching stripe " + stripeKey.stripeIx
- + " metadata with includes: " + DebugUtils.toString(stripeIncludes));
+ if (LlapIoImpl.ORC_LOGGER.isTraceEnabled()) {
+ LlapIoImpl.ORC_LOGGER.trace("Caching stripe {} metadata with includes: {}",
+ stripeKey.stripeIx, DebugUtils.toString(stripeIncludes));
}
}
}
consumer.setStripeMetadata(stripeMetadata);
}
if (!stripeMetadata.hasAllIndexes(stripeIncludes)) {
- if (DebugUtils.isTraceOrcEnabled()) {
- LlapIoImpl.LOG.info("Updating indexes in stripe " + stripeKey.stripeIx
- + " metadata for includes: " + DebugUtils.toString(stripeIncludes));
+ if (LlapIoImpl.ORC_LOGGER.isTraceEnabled()) {
+ LlapIoImpl.ORC_LOGGER.trace("Updating indexes in stripe {} metadata for includes: {}",
+ stripeKey.stripeIx, DebugUtils.toString(stripeIncludes));
}
assert isFoundInCache;
counters.incrCounter(LlapIOCounters.METADATA_CACHE_MISS);
@@ -432,9 +426,8 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void>
// Done with all the things.
recordReaderTime(startTime);
consumer.setDone();
- if (DebugUtils.isTraceMttEnabled()) {
- LlapIoImpl.LOG.info("done processing " + split);
- }
+
+ LlapIoImpl.LOG.trace("done processing {}", split);
// Close the stripe reader, we are done reading.
cleanupReaders();
@@ -584,9 +577,7 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void>
if (fileKey instanceof Long && HiveConf.getBoolVar(conf, ConfVars.LLAP_IO_USE_FILEID_PATH)) {
path = HdfsUtils.getFileIdPath(fs, path, (long)fileKey);
}
- if (DebugUtils.isTraceOrcEnabled()) {
- LOG.info("Creating reader for " + path + " (" + split.getPath() + ")");
- }
+ LlapIoImpl.ORC_LOGGER.trace("Creating reader for {} ({})", path, split.getPath());
long startTime = counters.startTimeCounter();
ReaderOptions opts = OrcFile.readerOptions(conf).filesystem(fs).fileMetadata(fileMetadata);
orcReader = EncodedOrcFile.createReader(path, opts);
@@ -640,17 +631,17 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void>
counters.incrTimeCounter(LlapIOCounters.HDFS_TIME_NS, startTime);
if (hasFileId && metadataCache != null) {
value = metadataCache.putStripeMetadata(value);
- if (DebugUtils.isTraceOrcEnabled()) {
- LlapIoImpl.LOG.info("Caching stripe " + stripeKey.stripeIx
- + " metadata with includes: " + DebugUtils.toString(globalInc));
+ if (LlapIoImpl.ORC_LOGGER.isTraceEnabled()) {
+ LlapIoImpl.ORC_LOGGER.trace("Caching stripe {} metadata with includes: {}",
+ stripeKey.stripeIx, DebugUtils.toString(globalInc));
}
}
}
// We might have got an old value from cache; recheck it has indexes.
if (!value.hasAllIndexes(globalInc)) {
- if (DebugUtils.isTraceOrcEnabled()) {
- LlapIoImpl.LOG.info("Updating indexes in stripe " + stripeKey.stripeIx
- + " metadata for includes: " + DebugUtils.toString(globalInc));
+ if (LlapIoImpl.ORC_LOGGER.isTraceEnabled()) {
+ LlapIoImpl.ORC_LOGGER.trace("Updating indexes in stripe {} metadata for includes: {}",
+ stripeKey.stripeIx, DebugUtils.toString(globalInc));
}
updateLoadedIndexes(value, si, globalInc, sargColumns);
}
@@ -677,9 +668,9 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void>
if (datas == null) continue;
for (ColumnStreamData data : datas) {
if (data == null || data.decRef() != 0) continue;
- if (DebugUtils.isTraceLockingEnabled()) {
+ if (LlapIoImpl.LOCKING_LOGGER.isTraceEnabled()) {
for (MemoryBuffer buf : data.getCacheBuffers()) {
- LlapIoImpl.LOG.info("Unlocking " + buf + " at the end of processing");
+ LlapIoImpl.LOCKING_LOGGER.trace("Unlocking {} at the end of processing", buf);
}
}
bufferManager.decRefBuffers(data.getCacheBuffers());
@@ -718,14 +709,14 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void>
boolean isNone = rgsToRead == SargApplier.READ_NO_RGS,
isAll = rgsToRead == SargApplier.READ_ALL_RGS;
hasAnyData = hasAnyData || !isNone;
- if (DebugUtils.isTraceOrcEnabled()) {
+ if (LlapIoImpl.ORC_LOGGER.isTraceEnabled()) {
if (isNone) {
- LlapIoImpl.LOG.info("SARG eliminated all RGs for stripe " + stripeIx);
+ LlapIoImpl.ORC_LOGGER.trace("SARG eliminated all RGs for stripe {}", stripeIx);
} else if (!isAll) {
- LlapIoImpl.LOG.info("SARG picked RGs for stripe " + stripeIx + ": "
- + DebugUtils.toString(rgsToRead));
+ LlapIoImpl.ORC_LOGGER.trace("SARG picked RGs for stripe {}: {}",
+ stripeIx, DebugUtils.toString(rgsToRead));
} else {
- LlapIoImpl.LOG.info("Will read all " + rgCount + " RGs for stripe " + stripeIx);
+ LlapIoImpl.ORC_LOGGER.trace("Will read all {} RGs for stripe {}", rgCount, stripeIx);
}
}
assert isAll || isNone || rgsToRead.length == rgCount;
@@ -768,12 +759,12 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void>
long offset = split.getStart(), maxOffset = offset + split.getLength();
stripeIxFrom = -1;
int stripeIxTo = -1;
- if (LlapIoImpl.LOGL.isDebugEnabled()) {
+ if (LlapIoImpl.ORC_LOGGER.isDebugEnabled()) {
String tmp = "FileSplit {" + split.getStart() + ", " + split.getLength() + "}; stripes ";
for (StripeInformation stripe : stripes) {
tmp += "{" + stripe.getOffset() + ", " + stripe.getLength() + "}, ";
}
- LlapIoImpl.LOG.debug(tmp);
+ LlapIoImpl.ORC_LOGGER.debug(tmp);
}
int stripeIx = 0;
@@ -785,33 +776,25 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void>
continue;
}
if (stripeIxFrom == -1) {
- if (DebugUtils.isTraceOrcEnabled()) {
- LlapIoImpl.LOG.info("Including stripes from " + stripeIx
- + " (" + stripeStart + " >= " + offset + ")");
- }
+ LlapIoImpl.ORC_LOGGER.trace("Including stripes from {} ({} >= {})",
+ stripeIx, stripeStart, offset);
stripeIxFrom = stripeIx;
}
if (stripeStart >= maxOffset) {
stripeIxTo = stripeIx;
- if (DebugUtils.isTraceOrcEnabled()) {
- LlapIoImpl.LOG.info("Including stripes until " + stripeIxTo + " (" + stripeStart
- + " >= " + maxOffset + "); " + (stripeIxTo - stripeIxFrom) + " stripes");
- }
+ LlapIoImpl.ORC_LOGGER.trace("Including stripes until {} ({} >= {}); {} stripes",
+ stripeIxTo, stripeStart, maxOffset, (stripeIxTo - stripeIxFrom));
break;
}
++stripeIx;
}
if (stripeIxFrom == -1) {
- if (LlapIoImpl.LOG.isInfoEnabled()) {
- LlapIoImpl.LOG.info("Not including any stripes - empty split");
- }
+ LlapIoImpl.LOG.info("Not including any stripes - empty split");
}
if (stripeIxTo == -1 && stripeIxFrom != -1) {
stripeIxTo = stripeIx;
- if (DebugUtils.isTraceOrcEnabled()) {
- LlapIoImpl.LOG.info("Including stripes until " + stripeIx + " (end of file); "
- + (stripeIxTo - stripeIxFrom) + " stripes");
- }
+ LlapIoImpl.ORC_LOGGER.trace("Including stripes until {} (end of file); {} stripes",
+ stripeIx, (stripeIxTo - stripeIxFrom));
}
readState = new boolean[stripeIxTo - stripeIxFrom][][];
}
@@ -869,9 +852,9 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void>
long startTime = counters.startTimeCounter();
DiskRangeList result = orcDataReader.readFileData(range, baseOffset, doForceDirect);
counters.recordHdfsTime(startTime);
- if (DebugUtils.isTraceOrcEnabled() && LOG.isInfoEnabled()) {
- LOG.info("Disk ranges after disk read (file " + fileKey + ", base offset " + baseOffset
- + "): " + RecordReaderUtils.stringifyDiskRanges(result));
+ if (LlapIoImpl.ORC_LOGGER.isTraceEnabled()) {
+ LlapIoImpl.ORC_LOGGER.trace("Disk ranges after disk read (file {}, base offset {}): {}",
+ fileKey, baseOffset, RecordReaderUtils.stringifyDiskRanges(result));
}
return result;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/3b6b56d7/llap-server/src/java/org/apache/hadoop/hive/llap/old/BufferInProgress.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/old/BufferInProgress.java b/llap-server/src/java/org/apache/hadoop/hive/llap/old/BufferInProgress.java
deleted file mode 100644
index 9782b81..0000000
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/old/BufferInProgress.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.llap.old;
-
-import org.apache.hadoop.hive.llap.old.BufferPool.WeakBuffer;
-import org.apache.hadoop.hive.llap.old.ChunkPool.Chunk;
-
-/**
- * Helper struct that is used by loaders (e.g. OrcLoader) and chunk writer to write chunks.
- */
-public class BufferInProgress {
- /** Buffer that is being written to. */
- public final WeakBuffer buffer;
- /** Offset in buffer where writing can proceed */
- public int offset; // TODO: use WB's position; these have separate lifecycle now, needed?
- private final int bufferLimit;
-
- /** The chunk that is currently being written. */
- private Chunk chunkInProgress = null;
- /** The row count of the chunk currently being written. */
- private int chunkInProgressRows = 0;
-
- public BufferInProgress(WeakBuffer buffer) {
- this.buffer = buffer;
- this.bufferLimit = buffer.getContents().limit();
- this.offset = 0;
- }
-
- public Chunk ensureChunk() {
- if (chunkInProgress == null) {
- chunkInProgress = new Chunk(buffer, offset, 0);
- chunkInProgressRows = 0;
- }
- return chunkInProgress;
- }
-
- public Chunk extractChunk() {
- Chunk result = chunkInProgress;
- chunkInProgress = null;
- chunkInProgressRows = 0;
- return result;
- }
-
- public void update(int newOffset, int rowsWritten) {
- if (newOffset > bufferLimit) {
- throw new AssertionError("Offset is beyond buffer limit: " + newOffset + "/" + bufferLimit
- + "; previous offset " + offset + ", chunk " + chunkInProgress);
- }
- chunkInProgress.length += (newOffset - offset);
- this.offset = newOffset;
- this.chunkInProgressRows += rowsWritten;
- }
-
- public int getChunkInProgressRows() {
- return chunkInProgressRows;
- }
-
- public int getSpaceLeft() {
- return getSpaceLeft(-1);
- }
-
- public int getSpaceLeft(int offset) {
- offset = (offset >= 0) ? offset : this.offset;
- return buffer.getContents().limit() - offset;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/3b6b56d7/llap-server/src/java/org/apache/hadoop/hive/llap/old/BufferPool.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/old/BufferPool.java b/llap-server/src/java/org/apache/hadoop/hive/llap/old/BufferPool.java
deleted file mode 100644
index fc10b2b..0000000
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/old/BufferPool.java
+++ /dev/null
@@ -1,225 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.hadoop.hive.llap.old;
-
-import java.nio.ByteBuffer;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.llap.DebugUtils;
-import org.apache.hadoop.hive.llap.io.api.impl.LlapIoImpl;
-
-import com.google.common.annotations.VisibleForTesting;
-
-public class BufferPool {
- // TODO: we should keep evicted buffers for reuse. Perhaps that too should be factored out.
- private final CachePolicy cachePolicy;
- private final Object evictionNotifyObj = new Object();
- private int evictionIsWaiting; // best effort flag
- private final long maxCacheSize;
- private final int bufferSize;
-
-
- public BufferPool(Configuration conf) {
- this.maxCacheSize = 0;// HiveConf.getLongVar(conf, HiveConf.ConfVars.LLAP_CACHE_SIZE);
- this.bufferSize = 0; // HiveConf.getIntVar(conf, HiveConf.ConfVars.LLAP_BUFFER_SIZE);
- this.cachePolicy = null;
- }
-
- /**
- * Allocates a new buffer. Buffer starts out locked (assumption is that caller is going to
- * write to it immediately and then unlock it; future writers/readers will lock and unlock).
- * @return Buffer.
- */
- public WeakBuffer allocateBuffer() throws InterruptedException {
- // TODO: for now, dumb byte arrays. Should be off-heap.
- ByteBuffer newBuffer = ByteBuffer.allocate(bufferSize);
- WeakBuffer wb = new WeakBuffer(this, newBuffer);
- // Don't touch the buffer - it's not in cache yet. cache() will set the initial priority.
- if (!wb.lock(false)) {
- throw new AssertionError("Cannot lock a new buffer");
- }
- if (DebugUtils.isTraceLockingEnabled()) {
- LlapIoImpl.LOG.info("Locked " + wb + " after creation");
- }
- boolean hasWaited = false;
- WeakBuffer evicted = null;
- while (true) {
- evicted = cachePolicy.cache(wb);
- if (evicted != CachePolicy.CANNOT_EVICT) break;
- if (DebugUtils.isTraceCachingEnabled() && !hasWaited) {
- LlapIoImpl.LOG.info("Failed to add a new block to cache; waiting for blocks to be unlocked");
- hasWaited = true;
- }
- synchronized (evictionNotifyObj) {
- ++evictionIsWaiting;
- evictionNotifyObj.wait(1000);
- --evictionIsWaiting;
- }
- }
- if (DebugUtils.isTraceCachingEnabled() && hasWaited) {
- LlapIoImpl.LOG.info("Eviction is done waiting");
- }
- if (evicted != null) {
- //if (evictionListener != null) {
- // evictionListener.evictionNotice(evicted);
- //}
- // After eviction notice, the contents can be reset.
- evicted.clear();
- }
- return wb;
- }
-
- private final void unblockEviction() {
- if (evictionIsWaiting <= 0) return;
- synchronized (evictionNotifyObj) {
- if (evictionIsWaiting <= 0) return;
- if (DebugUtils.isTraceCachingEnabled()) {
- LlapIoImpl.LOG.info("Notifying eviction that some block has been unlocked");
- }
- evictionNotifyObj.notifyAll();
- }
- }
-
- @VisibleForTesting
- public static WeakBuffer allocateFake() {
- return new WeakBuffer(null, ByteBuffer.wrap(new byte[1]));
- }
-
- /**
- * This class serves 3 purposes:
- * 1) it implements BufferPool-specific hashCode and equals (ByteBuffer ones are content-based);
- * 2) it contains the refCount;
- * 3) by extension from (2), it can be held while it is evicted; when locking before the usage,
- * the fact that the data has been evicted will be discovered (similar to weak_ptr).
- * Note: not static because when we wait for something to become evict-able,
- * we need to receive notifications from unlock (see unlock). Otherwise could be static.
- */
- public static final class WeakBuffer {
- private static final int EVICTED_REFCOUNT = -1;
- private final BufferPool parent;
- private ByteBuffer contents;
- private final AtomicInteger refCount = new AtomicInteger(0);
-
- // TODO: Fields pertaining to cache policy. Perhaps they should live in separate object.
- public double priority;
- public long lastUpdate = -1;
- public int indexInHeap = -1;
- public boolean isLockedInHeap = false;
-
- private WeakBuffer(BufferPool parent, ByteBuffer contents) {
- this.parent = parent;
- this.contents = contents;
- }
-
- public ByteBuffer getContents() {
- assert isLocked() : "Cannot get contents with refCount " + refCount.get();
- return contents;
- }
-
- @Override
- public int hashCode() {
- if (contents == null) return 0;
- return System.identityHashCode(contents);
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj) return true;
- if (!(obj instanceof WeakBuffer)) return false;
- // We only compare objects, and not contents of the ByteBuffer.
- // One ByteBuffer is never put in multiple WeakBuffer-s (that is the invariant).
- return contents == ((WeakBuffer)obj).contents;
- }
-
- public boolean lock(boolean doTouch) {
- int oldRefCount = -1;
- while (true) {
- oldRefCount = refCount.get();
- if (oldRefCount == EVICTED_REFCOUNT) return false;
- assert oldRefCount >= 0;
- if (refCount.compareAndSet(oldRefCount, oldRefCount + 1)) break;
- }
- if (doTouch && oldRefCount == 0 && parent != null) {
- parent.cachePolicy.notifyLock(this);
- }
- return true;
- }
-
- public boolean isLocked() {
- // Best-effort check. We cannot do a good check against caller thread, since
- // refCount could still be > 0 if someone else locked. This is used for asserts.
- return refCount.get() > 0;
- }
-
- public boolean isInvalid() {
- return refCount.get() == EVICTED_REFCOUNT;
- }
-
- public boolean isCleared() {
- return contents == null;
- }
-
- public void unlock() {
- int newRefCount = refCount.decrementAndGet();
- if (newRefCount < 0) {
- throw new AssertionError("Unexpected refCount " + newRefCount);
- }
- // If this block became eligible, see if we need to unblock the eviction.
- if (newRefCount == 0 && parent != null) {
- parent.cachePolicy.notifyUnlock(this);
- parent.unblockEviction();
- }
- }
-
- @Override
- public String toString() {
- return "0x" + Integer.toHexString(hashCode());
- }
-
- /**
- * @return Whether the we can invalidate; false if locked or already evicted.
- */
- boolean invalidate() {
- while (true) {
- int value = refCount.get();
- if (value != 0) return false;
- if (refCount.compareAndSet(value, EVICTED_REFCOUNT)) break;
- }
- if (DebugUtils.isTraceLockingEnabled()) {
- LlapIoImpl.LOG.info("Invalidated " + this + " due to eviction");
- }
- return true;
- }
-
- ByteBuffer clear() {
- assert refCount.get() == EVICTED_REFCOUNT;
- ByteBuffer result = contents;
- contents = null;
- return result;
- }
-
- public String toStringForCache() {
- return "[" + Integer.toHexString(hashCode()) + " " + String.format("%1$.2f", priority) + " "
- + lastUpdate + " " + (isLocked() ? "!" : ".") + "]";
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/3b6b56d7/llap-server/src/java/org/apache/hadoop/hive/llap/old/CachePolicy.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/old/CachePolicy.java b/llap-server/src/java/org/apache/hadoop/hive/llap/old/CachePolicy.java
deleted file mode 100644
index cca42fe..0000000
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/old/CachePolicy.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.llap.old;
-
-import org.apache.hadoop.hive.llap.old.BufferPool.WeakBuffer;
-
-public interface CachePolicy {
- public static final WeakBuffer CANNOT_EVICT = BufferPool.allocateFake();
-
- /**
- * @param buffer Buffer to cache.
- * @return Evicted buffer. All buffers are of the same size currently, so it is one or none.
- * It can also be CANNOT_EVICT fake buffer, if we cannot evict and thus cache.
- */
- WeakBuffer cache(WeakBuffer buffer);
- void notifyLock(WeakBuffer buffer);
- void notifyUnlock(WeakBuffer buffer);
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/3b6b56d7/llap-server/src/java/org/apache/hadoop/hive/llap/old/ChunkPool.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/old/ChunkPool.java b/llap-server/src/java/org/apache/hadoop/hive/llap/old/ChunkPool.java
deleted file mode 100644
index 4f9f165..0000000
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/old/ChunkPool.java
+++ /dev/null
@@ -1,237 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.llap.old;
-
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.hadoop.hive.llap.DebugUtils;
-import org.apache.hadoop.hive.llap.io.api.impl.LlapIoImpl;
-import org.apache.hadoop.hive.llap.old.BufferPool.WeakBuffer;
-import org.apache.hadoop.hive.llap.old.ChunkPool.Chunk;
-
-/**
- * This class contains the mapping of file chunks to buffers inside BufferPool.
- */
-public class ChunkPool<K> /*implements EvictionListener*/ {
- private final ConcurrentHashMap<K, Chunk> chunkCache = new ConcurrentHashMap<K, Chunk>();
-
- /** Number of unprocessed evictions, for the background thread. */
- private final AtomicInteger newEvictions = new AtomicInteger(0);
- private final Thread cleanupThread;
-
- public ChunkPool() {
- cleanupThread = new CleanupThread();
- cleanupThread.start();
- }
-
- /**
- * Gets a chunk from cache
- * TODO: We expect that in most cases, some related chunks (e.g. columns for a stripe)
- * will be stored in the same buffer. We could use this to get keys more efficiently.
- * On the other hand, real stripes are pretty big.
- * @param key key to search for.
- * @return Chunk corresponding to k.
- */
- public Chunk getChunk(K key, HashSet<WeakBuffer> lockedBuffers) {
- while (true) {
- Chunk result = chunkCache.get(key);
- if (result == null) return null;
- if (lockChunk(result, lockedBuffers)) return result;
- if (chunkCache.remove(key, result)) return null;
- }
- }
-
- private boolean lockChunk(Chunk result, HashSet<WeakBuffer> lockedBuffers) {
- // We expect the chain to have 1 or 2 buffers (2 if we are on buffer boundary). Keep track of
- // what we lock in the bitmask; may need fixing (extremely unlikely - 64+ buffer, giant chunks)
- boolean failedToLock = false;
- long blocksToUnlock = 0;
- long bit = 1 << 63; // The bit indicating that current chunk was locked.
-
- Chunk chunk = result;
- while (chunk != null) {
- if (lockedBuffers.contains(chunk.buffer)) {
- assert chunk.buffer.isLocked() : chunk.buffer + " is in lockedBuffers but is not locked";
- } else if (chunk.buffer.lock(true)) {
- if (DebugUtils.isTraceLockingEnabled()) {
- LlapIoImpl.LOG.info("Locked " + chunk.buffer + " for " + result);
- }
- lockedBuffers.add(chunk.buffer);
- blocksToUnlock += bit;
- } else {
- failedToLock = true;
- break;
- }
- bit >>>= 1;
- chunk = chunk.nextChunk;
- if (bit == 1 && chunk != null) {
- throw new AssertionError("Chunk chain was too long");
- }
- }
- if (!failedToLock) return true;
-
- bit = 1 << 63;
- Chunk chunk2 = result;
- while (chunk2 != chunk) {
- if ((blocksToUnlock & bit) == bit) {
- if (DebugUtils.isTraceLockingEnabled()) {
- LlapIoImpl.LOG.info("Unlocking " + chunk2.buffer + " due to failed chunk lock");
- }
- lockedBuffers.remove(chunk2.buffer);
- chunk2.buffer.unlock();
- }
- bit >>>= 1;
- chunk2 = chunk2.nextChunk;
- }
- return false;
- }
-
- private boolean verifyChunk(Chunk entry) {
- Chunk chunk = entry;
- while (chunk != null) {
- if (!chunk.buffer.lock(false)) break;
- chunk = chunk.nextChunk;
- }
- Chunk chunk2 = entry;
- while (chunk2 != chunk) {
- chunk2.buffer.unlock();
- chunk2 = chunk2.nextChunk;
- }
- return chunk == null;
- }
-
- public Chunk addOrGetChunk(K key, Chunk val, HashSet<WeakBuffer> lockedBuffers) {
- assert val.buffer.isLocked();
- while (true) {
- Chunk oldVal = chunkCache.putIfAbsent(key, val);
- if (oldVal == null) return val;
- if (DebugUtils.isTraceCachingEnabled()) {
- LlapIoImpl.LOG.info("Trying to cache when the chunk is already cached for "
- + key + "; old " + oldVal + ", new " + val);
- }
- if (lockChunk(oldVal, lockedBuffers)) return oldVal;
- // We found some old value but couldn't lock it; remove it.
- chunkCache.remove(key, oldVal);
- }
- }
-
- //@Override
- public void evictionNotice(WeakBuffer evicted) {
- int oldValue = newEvictions.getAndIncrement();
- if (oldValue == 0) {
- synchronized (newEvictions) {
- newEvictions.notifyAll();
- }
- }
- }
-
- public static class Chunk {
- public WeakBuffer buffer;
- public int offset, length;
- public Chunk nextChunk;
-
- public Chunk(WeakBuffer buffer, int offset, int length) {
- this.buffer = buffer;
- this.offset = offset;
- this.length = length;
- }
-
- public Chunk addChunk(Chunk another) {
- // Traversing list is bad; however, we expect that this will very rarely happen; and in
- // nearly all the cases when it does (buffer boundary) the list will have 1 element.
- Chunk chunk = this;
- while (chunk.nextChunk != null) {
- chunk = chunk.nextChunk;
- }
- chunk.nextChunk = another;
- return this;
- }
-
- @Override
- public String toString() {
- return "{" + buffer + ", " + offset + ", " + length + "}";
- }
-
- public String toFullString() {
- String result = "";
- Chunk chunk = this;
- while (chunk != null) {
- result += chunk.toString() + ", ";
- chunk = chunk.nextChunk;
- }
- return result;
- }
- }
-
- private final class CleanupThread extends Thread {
- private int APPROX_CLEANUP_INTERVAL_SEC = 600;
-
- public CleanupThread() {
- super("Llap ChunkPool cleanup thread");
- setDaemon(true);
- setPriority(1);
- }
-
- @Override
- public void run() {
- while (true) {
- try {
- doOneCleanupRound();
- } catch (InterruptedException ex) {
- LlapIoImpl.LOG.warn("Cleanup thread has been interrupted");
- Thread.currentThread().interrupt();
- break;
- } catch (Throwable t) {
- LlapIoImpl.LOG.error("Cleanup has failed; the thread will now exit", t);
- break;
- }
- }
- }
-
- private void doOneCleanupRound() throws InterruptedException {
- while (true) {
- int evictionsSinceLast = newEvictions.getAndSet(0);
- if (evictionsSinceLast > 0) break;
- synchronized (newEvictions) {
- newEvictions.wait(10000);
- }
- }
- // Duration is an estimate; if the size of the map changes rapidly, it can be very different.
- long endTime = System.nanoTime() + APPROX_CLEANUP_INTERVAL_SEC * 1000000000L;
- int processed = 0;
- // TODO: if this iterator affects the map in some bad way,
- // we'd need to sleep once per round instead.
- Iterator<Map.Entry<K, Chunk>> iter = chunkCache.entrySet().iterator();
- while (iter.hasNext()) {
- if (!verifyChunk(iter.next().getValue())) {
- iter.remove();
- }
- ++processed;
- int approxElementsLeft = chunkCache.size() - processed;
- Thread.sleep((approxElementsLeft <= 0)
- ? 1 : (endTime - System.nanoTime()) / (1000000L * approxElementsLeft));
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/3b6b56d7/llap-server/src/main/resources/llap-daemon-log4j2.properties
----------------------------------------------------------------------
diff --git a/llap-server/src/main/resources/llap-daemon-log4j2.properties b/llap-server/src/main/resources/llap-daemon-log4j2.properties
index 5051ca5..268eb59 100644
--- a/llap-server/src/main/resources/llap-daemon-log4j2.properties
+++ b/llap-server/src/main/resources/llap-daemon-log4j2.properties
@@ -64,7 +64,19 @@ appender.HISTORYAPPENDER.strategy.type = DefaultRolloverStrategy
appender.HISTORYAPPENDER.strategy.max = ${sys:llap.daemon.log.maxbackupindex}
# list of all loggers
-loggers = NIOServerCnxn, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX, HistoryLogger
+loggers = NIOServerCnxn, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX, HistoryLogger, LlapIoImpl, LlapIoOrc, LlapIoCache, LlapIoLocking
+
+logger.LlapIoImpl.name = LlapIoImpl
+logger.LlapIoImpl.level = INFO
+
+logger.LlapIoOrc.name = LlapIoOrc
+logger.LlapIoOrc.level = WARN
+
+logger.LlapIoCache.name = LlapIoCache
+logger.LlapIOCache.level = WARN
+
+logger.LlapIoLocking.name = LlapIoLocking
+logger.LlapIoLocking.level = WARN
logger.NIOServerCnxn.name = org.apache.zookeeper.server.NIOServerCnxn
logger.NIOServerCnxn.level = WARN
http://git-wip-us.apache.org/repos/asf/hive/blob/3b6b56d7/ql/src/java/org/apache/hadoop/hive/llap/DebugUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/llap/DebugUtils.java b/ql/src/java/org/apache/hadoop/hive/llap/DebugUtils.java
index ea626d7..3d81e43 100644
--- a/ql/src/java/org/apache/hadoop/hive/llap/DebugUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/llap/DebugUtils.java
@@ -23,49 +23,6 @@ package org.apache.hadoop.hive.llap;
* trace messages with low runtime cost, in order to investigate reproducible bugs.
*/
public class DebugUtils {
-
- public static boolean isTraceEnabled() {
- return false;
- }
-
- public static boolean isTraceOrcEnabled() {
- return false;
- }
-
- public static boolean isTraceLockingEnabled() {
- return false;
- }
-
- public static boolean isTraceMttEnabled() {
- return false;
- }
-
- public static boolean isTraceCachingEnabled() {
- return false;
- }
-
- public static String toString(long[] a, int offset, int len) {
- StringBuilder b = new StringBuilder();
- b.append('[');
- for (int i = offset; i < offset + len; ++i) {
- b.append(a[i]);
- b.append(", ");
- }
- b.append(']');
- return b.toString();
- }
-
- public static String toString(byte[] a, int offset, int len) {
- StringBuilder b = new StringBuilder();
- b.append('[');
- for (int i = offset; i < offset + len; ++i) {
- b.append(a[i]);
- b.append(", ");
- }
- b.append(']');
- return b.toString();
- }
-
public static String toString(boolean[] a) {
StringBuilder b = new StringBuilder();
b.append('[');
http://git-wip-us.apache.org/repos/asf/hive/blob/3b6b56d7/ql/src/java/org/apache/hadoop/hive/llap/LogLevels.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/llap/LogLevels.java b/ql/src/java/org/apache/hadoop/hive/llap/LogLevels.java
deleted file mode 100644
index 300230f..0000000
--- a/ql/src/java/org/apache/hadoop/hive/llap/LogLevels.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional debugrmation
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.llap;
-
-import org.slf4j.Logger;
-
-public class LogLevels {
- private final boolean isT, isD, isI, isW, isE;
-
- public LogLevels(Logger log) {
- isT = log.isTraceEnabled();
- isD = log.isDebugEnabled();
- isI = log.isInfoEnabled();
- isW = log.isWarnEnabled();
- isE = log.isErrorEnabled();
- }
-
- public boolean isTraceEnabled() {
- return isT;
- }
-
- public boolean isDebugEnabled() {
- return isD;
- }
-
- public boolean isInfoEnabled() {
- return isI;
- }
-
- public boolean isWarnEnabled() {
- return isW;
- }
-
- public boolean isErrorEnabled() {
- return isE;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/3b6b56d7/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java
index b8490df..31f5c72 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java
@@ -815,7 +815,7 @@ public class VectorGroupByOperator extends Operator<GroupByDesc> implements
aggregationBatchInfo = new VectorAggregationBufferBatch();
aggregationBatchInfo.compileAggregationBatchInfo(aggregators);
}
- LOG.warn("VectorGroupByOperator is vector output " + isVectorOutput);
+ LOG.info("VectorGroupByOperator is vector output {}", isVectorOutput);
outputObjInspector = ObjectInspectorFactory.getStandardStructObjectInspector(
outputFieldNames, objectInspectors);
if (isVectorOutput) {
http://git-wip-us.apache.org/repos/asf/hive/blob/3b6b56d7/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReader.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReader.java
index 96af96a..4d09dcd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReader.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReader.java
@@ -54,5 +54,5 @@ public interface EncodedReader {
* checks are entirely eliminated because this method is called with constant value, similar
* to just checking the constant in the first place.
*/
- void setDebugTracing(boolean isEnabled);
+ void setTracing(boolean isEnabled);
}
\ No newline at end of file
[23/50] [abbrv] hive git commit: HIVE-12992: Hive on tez: Bucket map
join plan is incorrect (Vikram Dixit K, reviewed by Jason Dere)
Posted by jd...@apache.org.
HIVE-12992: Hive on tez: Bucket map join plan is incorrect (Vikram Dixit K, reviewed by Jason Dere)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/761b5471
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/761b5471
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/761b5471
Branch: refs/heads/llap
Commit: 761b5471a0abbbb38ee35a715ea2d4e6d268d5a9
Parents: 7747458
Author: vikram <vi...@hortonworks.com>
Authored: Mon Mar 28 11:25:11 2016 -0700
Committer: vikram <vi...@hortonworks.com>
Committed: Mon Mar 28 11:37:32 2016 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/exec/OperatorUtils.java | 45 ++-
.../ql/optimizer/ReduceSinkMapJoinProc.java | 24 +-
.../clientpositive/bucket_map_join_tez1.q | 27 ++
.../llap/bucket_map_join_tez1.q.out | 308 +++++++++++++++++++
.../spark/bucket_map_join_tez1.q.out | 306 ++++++++++++++++++
.../tez/bucket_map_join_tez1.q.out | 294 ++++++++++++++++++
6 files changed, 985 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/761b5471/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java
index 3d664c1..41507b1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java
@@ -26,6 +26,7 @@ import java.util.Map;
import java.util.Set;
import org.apache.hadoop.hive.ql.exec.NodeUtils.Function;
+import org.apache.hadoop.hive.ql.plan.MapJoinDesc;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
import org.apache.hadoop.mapred.OutputCollector;
import org.slf4j.Logger;
@@ -80,6 +81,11 @@ public class OperatorUtils {
return found.size() == 1 ? found.iterator().next() : null;
}
+ public static <T> T findSingleOperatorUpstreamJoinAccounted(Operator<?> start, Class<T> clazz) {
+ Set<T> found = findOperatorsUpstreamJoinAccounted(start, clazz, new HashSet<T>());
+ return found.size() == 1 ? found.iterator().next(): null;
+ }
+
public static <T> Set<T> findOperatorsUpstream(Collection<Operator<?>> starts, Class<T> clazz) {
Set<T> found = new HashSet<T>();
for (Operator<?> start : starts) {
@@ -101,6 +107,34 @@ public class OperatorUtils {
return found;
}
+ public static <T> Set<T> findOperatorsUpstreamJoinAccounted(Operator<?> start, Class<T> clazz,
+ Set<T> found) {
+ if (clazz.isInstance(start)) {
+ found.add((T) start);
+ }
+ int onlyIncludeIndex = -1;
+ if (start instanceof AbstractMapJoinOperator) {
+ AbstractMapJoinOperator mapJoinOp = (AbstractMapJoinOperator) start;
+ MapJoinDesc desc = (MapJoinDesc) mapJoinOp.getConf();
+ onlyIncludeIndex = desc.getPosBigTable();
+ }
+ if (start.getParentOperators() != null) {
+ int i = 0;
+ for (Operator<?> parent : start.getParentOperators()) {
+ if (onlyIncludeIndex >= 0) {
+ if (onlyIncludeIndex == i) {
+ findOperatorsUpstream(parent, clazz, found);
+ }
+ } else {
+ findOperatorsUpstream(parent, clazz, found);
+ }
+ i++;
+ }
+ }
+ return found;
+ }
+
+
public static void setChildrenCollector(List<Operator<? extends OperatorDesc>> childOperators, OutputCollector out) {
if (childOperators == null) {
return;
@@ -202,7 +236,7 @@ public class OperatorUtils {
}
public static boolean sameRowSchema(Operator<?> operator1, Operator<?> operator2) {
- return operator1.getSchema().equals(operator2.getSchema());
+ return operator1.getSchema().equals(operator2.getSchema());
}
/**
@@ -220,9 +254,9 @@ public class OperatorUtils {
* them
*/
public static Multimap<Class<? extends Operator<?>>, Operator<?>> classifyOperators(
- Operator<?> start, Set<Class<? extends Operator<?>>> classes) {
+ Operator<?> start, Set<Class<? extends Operator<?>>> classes) {
ImmutableMultimap.Builder<Class<? extends Operator<?>>, Operator<?>> resultMap =
- new ImmutableMultimap.Builder<Class<? extends Operator<?>>, Operator<?>>();
+ new ImmutableMultimap.Builder<Class<? extends Operator<?>>, Operator<?>>();
List<Operator<?>> ops = new ArrayList<Operator<?>>();
ops.add(start);
while (!ops.isEmpty()) {
@@ -255,9 +289,9 @@ public class OperatorUtils {
* them
*/
public static Multimap<Class<? extends Operator<?>>, Operator<?>> classifyOperatorsUpstream(
- Operator<?> start, Set<Class<? extends Operator<?>>> classes) {
+ Operator<?> start, Set<Class<? extends Operator<?>>> classes) {
ImmutableMultimap.Builder<Class<? extends Operator<?>>, Operator<?>> resultMap =
- new ImmutableMultimap.Builder<Class<? extends Operator<?>>, Operator<?>>();
+ new ImmutableMultimap.Builder<Class<? extends Operator<?>>, Operator<?>>();
List<Operator<?>> ops = new ArrayList<Operator<?>>();
ops.add(start);
while (!ops.isEmpty()) {
@@ -296,5 +330,4 @@ public class OperatorUtils {
}
return numberOperators;
}
-
}
http://git-wip-us.apache.org/repos/asf/hive/blob/761b5471/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkMapJoinProc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkMapJoinProc.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkMapJoinProc.java
index 1e8f30e..00afc18 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkMapJoinProc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkMapJoinProc.java
@@ -220,8 +220,8 @@ public class ReduceSinkMapJoinProc implements NodeProcessor {
tableSize = 1;
}
LOG.info("Mapjoin " + mapJoinOp + "(bucket map join = )" + joinConf.isBucketMapJoin()
- + ", pos: " + pos + " --> " + parentWork.getName() + " (" + keyCount
- + " keys estimated from " + rowCount + " rows, " + bucketCount + " buckets)");
+ + ", pos: " + pos + " --> " + parentWork.getName() + " (" + keyCount
+ + " keys estimated from " + rowCount + " rows, " + bucketCount + " buckets)");
joinConf.getParentToInput().put(pos, parentWork.getName());
if (keyCount != Long.MAX_VALUE) {
joinConf.getParentKeyCounts().put(pos, keyCount);
@@ -247,10 +247,9 @@ public class ReduceSinkMapJoinProc implements NodeProcessor {
* 4. If we don't find a table scan operator, it has to be a reduce side operation.
*/
if (mapJoinWork == null) {
- Operator<?> rootOp =
- OperatorUtils.findSingleOperatorUpstream(
- mapJoinOp.getParentOperators().get(joinConf.getPosBigTable()),
- ReduceSinkOperator.class);
+ Operator<?> rootOp = OperatorUtils.findSingleOperatorUpstreamJoinAccounted(
+ mapJoinOp.getParentOperators().get(joinConf.getPosBigTable()),
+ ReduceSinkOperator.class);
if (rootOp == null) {
// likely we found a table scan operator
edgeType = EdgeType.CUSTOM_EDGE;
@@ -259,10 +258,9 @@ public class ReduceSinkMapJoinProc implements NodeProcessor {
edgeType = EdgeType.CUSTOM_SIMPLE_EDGE;
}
} else {
- Operator<?> rootOp =
- OperatorUtils.findSingleOperatorUpstream(
- mapJoinOp.getParentOperators().get(joinConf.getPosBigTable()),
- TableScanOperator.class);
+ Operator<?> rootOp = OperatorUtils.findSingleOperatorUpstreamJoinAccounted(
+ mapJoinOp.getParentOperators().get(joinConf.getPosBigTable()),
+ TableScanOperator.class);
if (rootOp != null) {
// likely we found a table scan operator
edgeType = EdgeType.CUSTOM_EDGE;
@@ -320,7 +318,7 @@ public class ReduceSinkMapJoinProc implements NodeProcessor {
context.linkOpWithWorkMap.put(mapJoinOp, linkWorkMap);
List<ReduceSinkOperator> reduceSinks
- = context.linkWorkWithReduceSinkMap.get(parentWork);
+ = context.linkWorkWithReduceSinkMap.get(parentWork);
if (reduceSinks == null) {
reduceSinks = new ArrayList<ReduceSinkOperator>();
}
@@ -358,7 +356,7 @@ public class ReduceSinkMapJoinProc implements NodeProcessor {
// let the dummy op be the parent of mapjoin op
mapJoinOp.replaceParent(parentRS, dummyOp);
List<Operator<? extends OperatorDesc>> dummyChildren =
- new ArrayList<Operator<? extends OperatorDesc>>();
+ new ArrayList<Operator<? extends OperatorDesc>>();
dummyChildren.add(mapJoinOp);
dummyOp.setChildOperators(dummyChildren);
dummyOperators.add(dummyOp);
@@ -384,4 +382,4 @@ public class ReduceSinkMapJoinProc implements NodeProcessor {
return true;
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/761b5471/ql/src/test/queries/clientpositive/bucket_map_join_tez1.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/bucket_map_join_tez1.q b/ql/src/test/queries/clientpositive/bucket_map_join_tez1.q
index 8ed630e..95585db 100644
--- a/ql/src/test/queries/clientpositive/bucket_map_join_tez1.q
+++ b/ql/src/test/queries/clientpositive/bucket_map_join_tez1.q
@@ -40,6 +40,33 @@ select count(*)
from
(select distinct key, value from tab_part) a join tab b on a.key = b.key;
+explain
+select count(*)
+from
+(select a.key as key, a.value as value from tab a join tab_part b on a.key = b.key) c
+join
+tab_part d on c.key = d.key;
+
+select count(*)
+from
+(select a.key as key, a.value as value from tab a join tab_part b on a.key = b.key) c
+join
+tab_part d on c.key = d.key;
+
+explain
+select count(*)
+from
+tab_part d
+join
+(select a.key as key, a.value as value from tab a join tab_part b on a.key = b.key) c on c.key = d.key;
+
+select count(*)
+from
+tab_part d
+join
+(select a.key as key, a.value as value from tab a join tab_part b on a.key = b.key) c on c.key = d.key;
+
+
-- one side is really bucketed. srcbucket_mapjoin is not really a bucketed table.
-- In this case the sub-query is chosen as the big table.
explain
http://git-wip-us.apache.org/repos/asf/hive/blob/761b5471/ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out b/ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out
index 21cfa5c..204da88 100644
--- a/ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out
+++ b/ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out
@@ -325,6 +325,314 @@ POSTHOOK: Input: default@tab_part
POSTHOOK: Input: default@tab_part@ds=2008-04-08
#### A masked pattern was here ####
242
+PREHOOK: query: explain
+select count(*)
+from
+(select a.key as key, a.value as value from tab a join tab_part b on a.key = b.key) c
+join
+tab_part d on c.key = d.key
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(*)
+from
+(select a.key as key, a.value as value from tab a join tab_part b on a.key = b.key) c
+join
+tab_part d on c.key = d.key
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Map 2 <- Map 1 (CUSTOM_EDGE), Map 4 (CUSTOM_EDGE)
+ Reducer 3 <- Map 2 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: llap
+ LLAP IO: no inputs
+ Map 2
+ Map Operator Tree:
+ TableScan
+ alias: b
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0
+ input vertices:
+ 0 Map 1
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ input vertices:
+ 1 Map 4
+ Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Execution mode: llap
+ LLAP IO: no inputs
+ Map 4
+ Map Operator Tree:
+ TableScan
+ alias: b
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: llap
+ LLAP IO: no inputs
+ Reducer 3
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(*)
+from
+(select a.key as key, a.value as value from tab a join tab_part b on a.key = b.key) c
+join
+tab_part d on c.key = d.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tab
+PREHOOK: Input: default@tab@ds=2008-04-08
+PREHOOK: Input: default@tab_part
+PREHOOK: Input: default@tab_part@ds=2008-04-08
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*)
+from
+(select a.key as key, a.value as value from tab a join tab_part b on a.key = b.key) c
+join
+tab_part d on c.key = d.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tab
+POSTHOOK: Input: default@tab@ds=2008-04-08
+POSTHOOK: Input: default@tab_part
+POSTHOOK: Input: default@tab_part@ds=2008-04-08
+#### A masked pattern was here ####
+1166
+PREHOOK: query: explain
+select count(*)
+from
+tab_part d
+join
+(select a.key as key, a.value as value from tab a join tab_part b on a.key = b.key) c on c.key = d.key
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(*)
+from
+tab_part d
+join
+(select a.key as key, a.value as value from tab a join tab_part b on a.key = b.key) c on c.key = d.key
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Map 3 <- Map 1 (CUSTOM_EDGE), Map 2 (CUSTOM_EDGE)
+ Reducer 4 <- Map 3 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: d
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: llap
+ LLAP IO: no inputs
+ Map 2
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: llap
+ LLAP IO: no inputs
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: d
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0
+ input vertices:
+ 0 Map 2
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ input vertices:
+ 0 Map 1
+ Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Execution mode: llap
+ LLAP IO: no inputs
+ Reducer 4
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(*)
+from
+tab_part d
+join
+(select a.key as key, a.value as value from tab a join tab_part b on a.key = b.key) c on c.key = d.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tab
+PREHOOK: Input: default@tab@ds=2008-04-08
+PREHOOK: Input: default@tab_part
+PREHOOK: Input: default@tab_part@ds=2008-04-08
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*)
+from
+tab_part d
+join
+(select a.key as key, a.value as value from tab a join tab_part b on a.key = b.key) c on c.key = d.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tab
+POSTHOOK: Input: default@tab@ds=2008-04-08
+POSTHOOK: Input: default@tab_part
+POSTHOOK: Input: default@tab_part@ds=2008-04-08
+#### A masked pattern was here ####
+1166
PREHOOK: query: -- one side is really bucketed. srcbucket_mapjoin is not really a bucketed table.
-- In this case the sub-query is chosen as the big table.
explain
http://git-wip-us.apache.org/repos/asf/hive/blob/761b5471/ql/src/test/results/clientpositive/spark/bucket_map_join_tez1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/bucket_map_join_tez1.q.out b/ql/src/test/results/clientpositive/spark/bucket_map_join_tez1.q.out
index 4899c3a..2d66d35 100644
--- a/ql/src/test/results/clientpositive/spark/bucket_map_join_tez1.q.out
+++ b/ql/src/test/results/clientpositive/spark/bucket_map_join_tez1.q.out
@@ -326,6 +326,312 @@ POSTHOOK: Input: default@tab_part
POSTHOOK: Input: default@tab_part@ds=2008-04-08
#### A masked pattern was here ####
242
+PREHOOK: query: explain
+select count(*)
+from
+(select a.key as key, a.value as value from tab a join tab_part b on a.key = b.key) c
+join
+tab_part d on c.key = d.key
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(*)
+from
+(select a.key as key, a.value as value from tab a join tab_part b on a.key = b.key) c
+join
+tab_part d on c.key = d.key
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-2 is a root stage
+ Stage-1 depends on stages: Stage-2
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-2
+ Spark
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE
+ Spark HashTable Sink Operator
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ Local Work:
+ Map Reduce Local Work
+ Map 4
+ Map Operator Tree:
+ TableScan
+ alias: b
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Spark HashTable Sink Operator
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Spark
+ Edges:
+ Reducer 3 <- Map 2 (GROUP, 1)
+#### A masked pattern was here ####
+ Vertices:
+ Map 2
+ Map Operator Tree:
+ TableScan
+ alias: b
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0
+ input vertices:
+ 0 Map 1
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ input vertices:
+ 1 Map 4
+ Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Local Work:
+ Map Reduce Local Work
+ Reducer 3
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(*)
+from
+(select a.key as key, a.value as value from tab a join tab_part b on a.key = b.key) c
+join
+tab_part d on c.key = d.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tab
+PREHOOK: Input: default@tab@ds=2008-04-08
+PREHOOK: Input: default@tab_part
+PREHOOK: Input: default@tab_part@ds=2008-04-08
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*)
+from
+(select a.key as key, a.value as value from tab a join tab_part b on a.key = b.key) c
+join
+tab_part d on c.key = d.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tab
+POSTHOOK: Input: default@tab@ds=2008-04-08
+POSTHOOK: Input: default@tab_part
+POSTHOOK: Input: default@tab_part@ds=2008-04-08
+#### A masked pattern was here ####
+1166
+PREHOOK: query: explain
+select count(*)
+from
+tab_part d
+join
+(select a.key as key, a.value as value from tab a join tab_part b on a.key = b.key) c on c.key = d.key
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(*)
+from
+tab_part d
+join
+(select a.key as key, a.value as value from tab a join tab_part b on a.key = b.key) c on c.key = d.key
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-2 is a root stage
+ Stage-1 depends on stages: Stage-2
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-2
+ Spark
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: d
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Spark HashTable Sink Operator
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ Local Work:
+ Map Reduce Local Work
+ Map 2
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE
+ Spark HashTable Sink Operator
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Spark
+ Edges:
+ Reducer 4 <- Map 3 (GROUP, 1)
+#### A masked pattern was here ####
+ Vertices:
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: d
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0
+ input vertices:
+ 0 Map 2
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ input vertices:
+ 0 Map 1
+ Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Local Work:
+ Map Reduce Local Work
+ Reducer 4
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(*)
+from
+tab_part d
+join
+(select a.key as key, a.value as value from tab a join tab_part b on a.key = b.key) c on c.key = d.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tab
+PREHOOK: Input: default@tab@ds=2008-04-08
+PREHOOK: Input: default@tab_part
+PREHOOK: Input: default@tab_part@ds=2008-04-08
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*)
+from
+tab_part d
+join
+(select a.key as key, a.value as value from tab a join tab_part b on a.key = b.key) c on c.key = d.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tab
+POSTHOOK: Input: default@tab@ds=2008-04-08
+POSTHOOK: Input: default@tab_part
+POSTHOOK: Input: default@tab_part@ds=2008-04-08
+#### A masked pattern was here ####
+1166
PREHOOK: query: -- one side is really bucketed. srcbucket_mapjoin is not really a bucketed table.
-- In this case the sub-query is chosen as the big table.
explain
http://git-wip-us.apache.org/repos/asf/hive/blob/761b5471/ql/src/test/results/clientpositive/tez/bucket_map_join_tez1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/bucket_map_join_tez1.q.out b/ql/src/test/results/clientpositive/tez/bucket_map_join_tez1.q.out
index 2e10157..30c4107 100644
--- a/ql/src/test/results/clientpositive/tez/bucket_map_join_tez1.q.out
+++ b/ql/src/test/results/clientpositive/tez/bucket_map_join_tez1.q.out
@@ -315,6 +315,300 @@ POSTHOOK: Input: default@tab_part
POSTHOOK: Input: default@tab_part@ds=2008-04-08
#### A masked pattern was here ####
242
+PREHOOK: query: explain
+select count(*)
+from
+(select a.key as key, a.value as value from tab a join tab_part b on a.key = b.key) c
+join
+tab_part d on c.key = d.key
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(*)
+from
+(select a.key as key, a.value as value from tab a join tab_part b on a.key = b.key) c
+join
+tab_part d on c.key = d.key
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Map 2 <- Map 1 (CUSTOM_EDGE), Map 4 (CUSTOM_EDGE)
+ Reducer 3 <- Map 2 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE
+ Map 2
+ Map Operator Tree:
+ TableScan
+ alias: b
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0
+ input vertices:
+ 0 Map 1
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ input vertices:
+ 1 Map 4
+ Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Map 4
+ Map Operator Tree:
+ TableScan
+ alias: b
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reducer 3
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(*)
+from
+(select a.key as key, a.value as value from tab a join tab_part b on a.key = b.key) c
+join
+tab_part d on c.key = d.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tab
+PREHOOK: Input: default@tab@ds=2008-04-08
+PREHOOK: Input: default@tab_part
+PREHOOK: Input: default@tab_part@ds=2008-04-08
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*)
+from
+(select a.key as key, a.value as value from tab a join tab_part b on a.key = b.key) c
+join
+tab_part d on c.key = d.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tab
+POSTHOOK: Input: default@tab@ds=2008-04-08
+POSTHOOK: Input: default@tab_part
+POSTHOOK: Input: default@tab_part@ds=2008-04-08
+#### A masked pattern was here ####
+1166
+PREHOOK: query: explain
+select count(*)
+from
+tab_part d
+join
+(select a.key as key, a.value as value from tab a join tab_part b on a.key = b.key) c on c.key = d.key
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(*)
+from
+tab_part d
+join
+(select a.key as key, a.value as value from tab a join tab_part b on a.key = b.key) c on c.key = d.key
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Map 3 <- Map 1 (CUSTOM_EDGE), Map 2 (CUSTOM_EDGE)
+ Reducer 4 <- Map 3 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: d
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Map 2
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: d
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0
+ input vertices:
+ 0 Map 2
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ input vertices:
+ 0 Map 1
+ Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Reducer 4
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(*)
+from
+tab_part d
+join
+(select a.key as key, a.value as value from tab a join tab_part b on a.key = b.key) c on c.key = d.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tab
+PREHOOK: Input: default@tab@ds=2008-04-08
+PREHOOK: Input: default@tab_part
+PREHOOK: Input: default@tab_part@ds=2008-04-08
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*)
+from
+tab_part d
+join
+(select a.key as key, a.value as value from tab a join tab_part b on a.key = b.key) c on c.key = d.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tab
+POSTHOOK: Input: default@tab@ds=2008-04-08
+POSTHOOK: Input: default@tab_part
+POSTHOOK: Input: default@tab_part@ds=2008-04-08
+#### A masked pattern was here ####
+1166
PREHOOK: query: -- one side is really bucketed. srcbucket_mapjoin is not really a bucketed table.
-- In this case the sub-query is chosen as the big table.
explain
[12/50] [abbrv] hive git commit: HIVE-13151 : Clean up UGI objects in
FileSystem cache for transactions, ADDENDUM (Wei Zheng,
reviewed by Eugene Koifman)
Posted by jd...@apache.org.
HIVE-13151 : Clean up UGI objects in FileSystem cache for transactions, ADDENDUM (Wei Zheng, reviewed by Eugene Koifman)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4fabd038
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4fabd038
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4fabd038
Branch: refs/heads/llap
Commit: 4fabd038cf64b906a89726805958c43b97194291
Parents: 6bfec2e
Author: Wei Zheng <we...@apache.org>
Authored: Thu Mar 24 22:18:32 2016 -0700
Committer: Wei Zheng <we...@apache.org>
Committed: Thu Mar 24 22:18:32 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java | 5 +++--
.../java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java | 4 ++--
ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java | 5 +++--
3 files changed, 8 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/4fabd038/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java
index 4c31a49..23b1b7f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java
@@ -275,8 +275,9 @@ public class Cleaner extends CompactorThread {
try {
FileSystem.closeAllForUGI(ugi);
} catch (IOException exception) {
- LOG.error("Could not clean up file-system handles for UGI: " + ugi, exception + " for " +
- ci.getFullPartitionName()); }
+ LOG.error("Could not clean up file-system handles for UGI: " + ugi + " for " +
+ ci.getFullPartitionName(), exception);
+ }
}
txnHandler.markCleaned(ci);
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/hive/blob/4fabd038/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
index 98ebf53..abbe5d4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
@@ -235,8 +235,8 @@ public class Initiator extends CompactorThread {
try {
FileSystem.closeAllForUGI(ugi);
} catch (IOException exception) {
- LOG.error("Could not clean up file-system handles for UGI: " + ugi, exception + " for " +
- ci.getFullPartitionName());
+ LOG.error("Could not clean up file-system handles for UGI: " + ugi + " for " +
+ ci.getFullPartitionName(), exception);
}
return compactionType;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/4fabd038/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
index e21ca27..6238e2b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
@@ -175,8 +175,9 @@ public class Worker extends CompactorThread {
try {
FileSystem.closeAllForUGI(ugi);
} catch (IOException exception) {
- LOG.error("Could not clean up file-system handles for UGI: " + ugi, exception + " for " +
- ci.getFullPartitionName()); }
+ LOG.error("Could not clean up file-system handles for UGI: " + ugi + " for " +
+ ci.getFullPartitionName(), exception);
+ }
}
txnHandler.markCompacted(ci);
} catch (Exception e) {
[44/50] [abbrv] hive git commit: HIVE-12619: Switching the field
order within an array of structs causes the query to fail (Mohammad and Jimmy,
reviewed by Sergio)
Posted by jd...@apache.org.
HIVE-12619: Switching the field order within an array of structs causes the query to fail (Mohammad and Jimmy, reviewed by Sergio)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b431c278
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b431c278
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b431c278
Branch: refs/heads/llap
Commit: b431c2788cd37fc788acd48beaf31c403361c1f0
Parents: 20a8192
Author: Jimmy Xiang <jx...@apache.org>
Authored: Thu Mar 10 10:32:57 2016 -0800
Committer: Jimmy Xiang <jx...@apache.org>
Committed: Tue Mar 29 19:47:18 2016 -0700
----------------------------------------------------------------------
.../io/parquet/convert/HiveSchemaConverter.java | 10 +--
.../parquet/read/DataWritableReadSupport.java | 75 ++++++++++++--------
.../ql/io/parquet/serde/ParquetHiveSerDe.java | 11 +--
.../clientpositive/parquet_schema_evolution.q | 14 ++++
.../parquet_map_null.q.java1.8.out | 1 +
.../parquet_schema_evolution.q.out | 65 +++++++++++++++++
.../clientpositive/parquet_type_promotion.q.out | 2 +-
7 files changed, 131 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b431c278/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveSchemaConverter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveSchemaConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveSchemaConverter.java
index b01f21f..40f6256 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveSchemaConverter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveSchemaConverter.java
@@ -24,12 +24,10 @@ import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
-
import org.apache.parquet.schema.ConversionPatterns;
import org.apache.parquet.schema.GroupType;
import org.apache.parquet.schema.MessageType;
import org.apache.parquet.schema.OriginalType;
-import org.apache.parquet.schema.PrimitiveType;
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
import org.apache.parquet.schema.Type;
import org.apache.parquet.schema.Type.Repetition;
@@ -120,9 +118,10 @@ public class HiveSchemaConverter {
// An optional group containing a repeated anonymous group "bag", containing
// 1 anonymous element "array_element"
+ @SuppressWarnings("deprecation")
private static GroupType convertArrayType(final String name, final ListTypeInfo typeInfo) {
final TypeInfo subType = typeInfo.getListElementTypeInfo();
- return listWrapper(name, OriginalType.LIST, new GroupType(Repetition.REPEATED,
+ return new GroupType(Repetition.OPTIONAL, name, OriginalType.LIST, new GroupType(Repetition.REPEATED,
ParquetHiveSerDe.ARRAY.toString(), convertType("array_element", subType)));
}
@@ -143,9 +142,4 @@ public class HiveSchemaConverter {
typeInfo.getMapValueTypeInfo());
return ConversionPatterns.mapType(Repetition.OPTIONAL, name, keyType, valueType);
}
-
- private static GroupType listWrapper(final String name, final OriginalType originalType,
- final GroupType groupType) {
- return new GroupType(Repetition.OPTIONAL, name, originalType, groupType);
- }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/b431c278/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java
index 53f3b72..3e38cc7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java
@@ -23,24 +23,28 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.io.IOConstants;
import org.apache.hadoop.hive.ql.io.parquet.convert.DataWritableRecordConverter;
+import org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe;
import org.apache.hadoop.hive.ql.metadata.VirtualColumn;
import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
import org.apache.hadoop.io.ArrayWritable;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.util.StringUtils;
-
import org.apache.parquet.hadoop.api.InitContext;
import org.apache.parquet.hadoop.api.ReadSupport;
import org.apache.parquet.io.api.RecordMaterializer;
import org.apache.parquet.schema.GroupType;
import org.apache.parquet.schema.MessageType;
+import org.apache.parquet.schema.OriginalType;
+import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
import org.apache.parquet.schema.Type;
+import org.apache.parquet.schema.Type.Repetition;
import org.apache.parquet.schema.Types;
-import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
/**
*
@@ -107,43 +111,58 @@ public class DataWritableReadSupport extends ReadSupport<ArrayWritable> {
private static List<Type> getProjectedGroupFields(GroupType schema, List<String> colNames, List<TypeInfo> colTypes) {
List<Type> schemaTypes = new ArrayList<Type>();
- ListIterator columnIterator = colNames.listIterator();
+ ListIterator<String> columnIterator = colNames.listIterator();
while (columnIterator.hasNext()) {
TypeInfo colType = colTypes.get(columnIterator.nextIndex());
- String colName = (String) columnIterator.next();
+ String colName = columnIterator.next();
Type fieldType = getFieldTypeIgnoreCase(schema, colName);
- if (fieldType != null) {
- if (colType.getCategory() == ObjectInspector.Category.STRUCT) {
- if (fieldType.isPrimitive()) {
- throw new IllegalStateException("Invalid schema data type, found: PRIMITIVE, expected: STRUCT");
- }
-
- GroupType groupFieldType = fieldType.asGroupType();
-
- List<Type> groupFields = getProjectedGroupFields(
- groupFieldType,
- ((StructTypeInfo) colType).getAllStructFieldNames(),
- ((StructTypeInfo) colType).getAllStructFieldTypeInfos()
- );
-
- Type[] typesArray = groupFields.toArray(new Type[0]);
- schemaTypes.add(Types.buildGroup(groupFieldType.getRepetition())
- .addFields(typesArray)
- .named(fieldType.getName())
- );
- } else {
- schemaTypes.add(fieldType);
- }
- } else {
- // Add type for schema evolution
+ if (fieldType == null) {
schemaTypes.add(Types.optional(PrimitiveTypeName.BINARY).named(colName));
+ } else {
+ schemaTypes.add(getProjectedType(colType, fieldType));
}
}
return schemaTypes;
}
+ private static Type getProjectedType(TypeInfo colType, Type fieldType) {
+ switch (colType.getCategory()) {
+ case STRUCT:
+ List<Type> groupFields = getProjectedGroupFields(
+ fieldType.asGroupType(),
+ ((StructTypeInfo) colType).getAllStructFieldNames(),
+ ((StructTypeInfo) colType).getAllStructFieldTypeInfos()
+ );
+
+ Type[] typesArray = groupFields.toArray(new Type[0]);
+ return Types.buildGroup(fieldType.getRepetition())
+ .addFields(typesArray)
+ .named(fieldType.getName());
+ case LIST:
+ TypeInfo elemType = ((ListTypeInfo) colType).getListElementTypeInfo();
+ if (elemType.getCategory() == ObjectInspector.Category.STRUCT) {
+ Type subFieldType = fieldType.asGroupType().getType(0);
+ if (!subFieldType.isPrimitive()) {
+ String subFieldName = subFieldType.getName();
+ Text name = new Text(subFieldName);
+ if (name.equals(ParquetHiveSerDe.ARRAY) || name.equals(ParquetHiveSerDe.LIST)) {
+ subFieldType = new GroupType(Repetition.REPEATED, subFieldName,
+ getProjectedType(elemType, subFieldType.asGroupType().getType(0)));
+ } else {
+ subFieldType = getProjectedType(elemType, subFieldType);
+ }
+ return Types.buildGroup(Repetition.OPTIONAL).as(OriginalType.LIST).addFields(
+ subFieldType).named(fieldType.getName());
+ }
+ }
+ break;
+ default:
+ }
+ return fieldType;
+ }
+
/**
* Searchs column names by name on a given Parquet message schema, and returns its projected
* Parquet schema types.
http://git-wip-us.apache.org/repos/asf/hive/blob/b431c278/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java
index e1bf8e2..995b965 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java
@@ -28,7 +28,6 @@ import org.apache.hadoop.hive.serde2.io.ParquetHiveRecord;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-
import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
@@ -37,7 +36,6 @@ import org.apache.hadoop.io.ArrayWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.parquet.hadoop.ParquetOutputFormat;
-import org.apache.parquet.hadoop.ParquetWriter;
/**
*
@@ -51,10 +49,7 @@ public class ParquetHiveSerDe extends AbstractSerDe {
public static final Text MAP_VALUE = new Text("value");
public static final Text MAP = new Text("map");
public static final Text ARRAY = new Text("bag");
-
- // default compression type for parquet output format
- private static final String DEFAULTCOMPRESSION =
- ParquetWriter.DEFAULT_COMPRESSION_CODEC_NAME.name();
+ public static final Text LIST = new Text("list");
// Map precision to the number bytes needed for binary conversion.
public static final int PRECISION_TO_BYTE_COUNT[] = new int[38];
@@ -78,7 +73,6 @@ public class ParquetHiveSerDe extends AbstractSerDe {
private LAST_OPERATION status;
private long serializedSize;
private long deserializedSize;
- private String compressionType;
private ParquetHiveRecord parquetRow;
@@ -97,9 +91,6 @@ public class ParquetHiveSerDe extends AbstractSerDe {
final String columnNameProperty = tbl.getProperty(serdeConstants.LIST_COLUMNS);
final String columnTypeProperty = tbl.getProperty(serdeConstants.LIST_COLUMN_TYPES);
- // Get compression properties
- compressionType = tbl.getProperty(ParquetOutputFormat.COMPRESSION, DEFAULTCOMPRESSION);
-
if (columnNameProperty.length() == 0) {
columnNames = new ArrayList<String>();
} else {
http://git-wip-us.apache.org/repos/asf/hive/blob/b431c278/ql/src/test/queries/clientpositive/parquet_schema_evolution.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/parquet_schema_evolution.q b/ql/src/test/queries/clientpositive/parquet_schema_evolution.q
index 193400f..e767b81 100644
--- a/ql/src/test/queries/clientpositive/parquet_schema_evolution.q
+++ b/ql/src/test/queries/clientpositive/parquet_schema_evolution.q
@@ -24,5 +24,19 @@ CREATE TABLE NewStructFieldTable STORED AS PARQUET AS SELECT * FROM NewStructFie
DESCRIBE NewStructFieldTable;
SELECT * FROM NewStructFieldTable;
+-- test if the order of fields in array<struct<>> changes, it works fine
+
+DROP TABLE IF EXISTS schema_test;
+CREATE TABLE schema_test (msg array<struct<f1: string, f2: string, a: array<struct<a1: string, a2: string>>, b: array<struct<b1: int, b2: int>>>>) STORED AS PARQUET;
+INSERT INTO TABLE schema_test SELECT array(named_struct('f1', 'abc', 'f2', 'abc2', 'a', array(named_struct('a1', 'a1', 'a2', 'a2')),
+ 'b', array(named_struct('b1', 1, 'b2', 2)))) FROM NewStructField LIMIT 2;
+SELECT * FROM schema_test;
+set hive.metastore.disallow.incompatible.col.type.changes=false;
+-- Order of fields swapped
+ALTER TABLE schema_test CHANGE msg msg array<struct<a: array<struct<a2: string, a1: string>>, b: array<struct<b2: int, b1: int>>, f2: string, f1: string>>;
+reset hive.metastore.disallow.incompatible.col.type.changes;
+SELECT * FROM schema_test;
+
+DROP TABLE schema_test;
DROP TABLE NewStructField;
DROP TABLE NewStructFieldTable;
http://git-wip-us.apache.org/repos/asf/hive/blob/b431c278/ql/src/test/results/clientpositive/parquet_map_null.q.java1.8.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/parquet_map_null.q.java1.8.out b/ql/src/test/results/clientpositive/parquet_map_null.q.java1.8.out
index dd541a5..1462cc2 100644
--- a/ql/src/test/results/clientpositive/parquet_map_null.q.java1.8.out
+++ b/ql/src/test/results/clientpositive/parquet_map_null.q.java1.8.out
@@ -38,6 +38,7 @@ POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@avro_table
POSTHOOK: Output: database:default
POSTHOOK: Output: default@parquet_table
+POSTHOOK: Lineage: parquet_table.avreau_col_1 SIMPLE [(avro_table)avro_table.FieldSchema(name:avreau_col_1, type:map<string,string>, comment:), ]
PREHOOK: query: SELECT * FROM parquet_table
PREHOOK: type: QUERY
PREHOOK: Input: default@parquet_table
http://git-wip-us.apache.org/repos/asf/hive/blob/b431c278/ql/src/test/results/clientpositive/parquet_schema_evolution.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/parquet_schema_evolution.q.out b/ql/src/test/results/clientpositive/parquet_schema_evolution.q.out
index 0b88d84..07595d2 100644
--- a/ql/src/test/results/clientpositive/parquet_schema_evolution.q.out
+++ b/ql/src/test/results/clientpositive/parquet_schema_evolution.q.out
@@ -125,6 +125,71 @@ POSTHOOK: Input: default@newstructfieldtable
{"a1":{"k1":"v1"},"a2":{"e1":5,"e2":null},"a3":null} NULL
{"a1":{"k1":"v1"},"a2":{"e1":5,"e2":null},"a3":null} NULL
{"a1":{"k1":"v1"},"a2":{"e1":5,"e2":null},"a3":null} NULL
+PREHOOK: query: -- test if the order of fields in array<struct<>> changes, it works fine
+
+DROP TABLE IF EXISTS schema_test
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: -- test if the order of fields in array<struct<>> changes, it works fine
+
+DROP TABLE IF EXISTS schema_test
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE schema_test (msg array<struct<f1: string, f2: string, a: array<struct<a1: string, a2: string>>, b: array<struct<b1: int, b2: int>>>>) STORED AS PARQUET
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@schema_test
+POSTHOOK: query: CREATE TABLE schema_test (msg array<struct<f1: string, f2: string, a: array<struct<a1: string, a2: string>>, b: array<struct<b1: int, b2: int>>>>) STORED AS PARQUET
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@schema_test
+PREHOOK: query: INSERT INTO TABLE schema_test SELECT array(named_struct('f1', 'abc', 'f2', 'abc2', 'a', array(named_struct('a1', 'a1', 'a2', 'a2')),
+ 'b', array(named_struct('b1', 1, 'b2', 2)))) FROM NewStructField LIMIT 2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@newstructfield
+PREHOOK: Output: default@schema_test
+POSTHOOK: query: INSERT INTO TABLE schema_test SELECT array(named_struct('f1', 'abc', 'f2', 'abc2', 'a', array(named_struct('a1', 'a1', 'a2', 'a2')),
+ 'b', array(named_struct('b1', 1, 'b2', 2)))) FROM NewStructField LIMIT 2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@newstructfield
+POSTHOOK: Output: default@schema_test
+POSTHOOK: Lineage: schema_test.msg EXPRESSION []
+PREHOOK: query: SELECT * FROM schema_test
+PREHOOK: type: QUERY
+PREHOOK: Input: default@schema_test
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM schema_test
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@schema_test
+#### A masked pattern was here ####
+[{"f1":"abc","f2":"abc2","a":[{"a1":"a1","a2":"a2"}],"b":[{"b1":1,"b2":2}]}]
+[{"f1":"abc","f2":"abc2","a":[{"a1":"a1","a2":"a2"}],"b":[{"b1":1,"b2":2}]}]
+PREHOOK: query: -- Order of fields swapped
+ALTER TABLE schema_test CHANGE msg msg array<struct<a: array<struct<a2: string, a1: string>>, b: array<struct<b2: int, b1: int>>, f2: string, f1: string>>
+PREHOOK: type: ALTERTABLE_RENAMECOL
+PREHOOK: Input: default@schema_test
+PREHOOK: Output: default@schema_test
+POSTHOOK: query: -- Order of fields swapped
+ALTER TABLE schema_test CHANGE msg msg array<struct<a: array<struct<a2: string, a1: string>>, b: array<struct<b2: int, b1: int>>, f2: string, f1: string>>
+POSTHOOK: type: ALTERTABLE_RENAMECOL
+POSTHOOK: Input: default@schema_test
+POSTHOOK: Output: default@schema_test
+PREHOOK: query: SELECT * FROM schema_test
+PREHOOK: type: QUERY
+PREHOOK: Input: default@schema_test
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM schema_test
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@schema_test
+#### A masked pattern was here ####
+[{"a":[{"a2":"a2","a1":"a1"}],"b":[{"b2":2,"b1":1}],"f2":"abc2","f1":"abc"}]
+[{"a":[{"a2":"a2","a1":"a1"}],"b":[{"b2":2,"b1":1}],"f2":"abc2","f1":"abc"}]
+PREHOOK: query: DROP TABLE schema_test
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@schema_test
+PREHOOK: Output: default@schema_test
+POSTHOOK: query: DROP TABLE schema_test
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@schema_test
+POSTHOOK: Output: default@schema_test
PREHOOK: query: DROP TABLE NewStructField
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@newstructfield
http://git-wip-us.apache.org/repos/asf/hive/blob/b431c278/ql/src/test/results/clientpositive/parquet_type_promotion.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/parquet_type_promotion.q.out b/ql/src/test/results/clientpositive/parquet_type_promotion.q.out
index 55f9b27..91c3fff 100644
--- a/ql/src/test/results/clientpositive/parquet_type_promotion.q.out
+++ b/ql/src/test/results/clientpositive/parquet_type_promotion.q.out
@@ -233,7 +233,7 @@ POSTHOOK: query: SELECT * FROM arrays_of_struct_to_map
POSTHOOK: type: QUERY
POSTHOOK: Input: default@arrays_of_struct_to_map
#### A masked pattern was here ####
-[{"c1":1}] [{"f2":77}]
+[{"c1":1}] [{"f2":88}]
PREHOOK: query: -- Testing schema evolution of adding columns into array<struct<>>
ALTER TABLE arrays_of_struct_to_map REPLACE COLUMNS (locations1 array<struct<c1:int,c2:int,c3:int>>, locations2
array<struct<f1:int,f2:int,f3:int>>)
[24/50] [abbrv] hive git commit: HIVE-13111: Fix timestamp /
interval_day_time wrong results with HIVE-9862 (Matt McCline,
reviewed by Jason Dere)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/serde/src/java/org/apache/hadoop/hive/serde2/io/TimestampWritable.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/io/TimestampWritable.java b/serde/src/java/org/apache/hadoop/hive/serde2/io/TimestampWritable.java
index fdc64e7..305fdbe 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/io/TimestampWritable.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/io/TimestampWritable.java
@@ -28,7 +28,6 @@ import java.text.SimpleDateFormat;
import java.util.Date;
import org.apache.hadoop.hive.common.type.HiveDecimal;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
import org.apache.hadoop.hive.serde2.ByteStream.RandomAccessOutput;
import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryUtils;
import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryUtils.VInt;
@@ -323,7 +322,20 @@ public class TimestampWritable implements WritableComparable<TimestampWritable>
return seconds + nanos / 1000000000;
}
+ public static long getLong(Timestamp timestamp) {
+ return timestamp.getTime() / 1000;
+ }
+ /**
+ *
+ * @return double representation of the timestamp, accurate to nanoseconds
+ */
+ public static double getDouble(Timestamp timestamp) {
+ double seconds, nanos;
+ seconds = millisToSeconds(timestamp.getTime());
+ nanos = timestamp.getNanos();
+ return seconds + nanos / 1000000000;
+ }
public void readFields(DataInput in) throws IOException {
in.readFully(internalBytes, 0, 4);
@@ -543,6 +555,21 @@ public class TimestampWritable implements WritableComparable<TimestampWritable>
return t;
}
+ public HiveDecimal getHiveDecimal() {
+ if (timestampEmpty) {
+ populateTimestamp();
+ }
+ return getHiveDecimal(timestamp);
+ }
+
+ public static HiveDecimal getHiveDecimal(Timestamp timestamp) {
+ // The BigDecimal class recommends not converting directly from double to BigDecimal,
+ // so we convert through a string...
+ Double timestampDouble = TimestampWritable.getDouble(timestamp);
+ HiveDecimal result = HiveDecimal.create(timestampDouble.toString());
+ return result;
+ }
+
/**
* Converts the time in seconds or milliseconds to a timestamp.
* @param time time in seconds or in milliseconds
@@ -553,6 +580,17 @@ public class TimestampWritable implements WritableComparable<TimestampWritable>
return new Timestamp(intToTimestampInSeconds ? time * 1000 : time);
}
+ /**
+ * Converts the time in seconds or milliseconds to a timestamp.
+ * @param time time in seconds or in milliseconds
+ * @return the timestamp
+ */
+ public static void setTimestampFromLong(Timestamp timestamp, long time,
+ boolean intToTimestampInSeconds) {
+ // If the time is in seconds, converts it to milliseconds first.
+ timestamp.setTime(intToTimestampInSeconds ? time * 1000 : time);
+ }
+
public static Timestamp doubleToTimestamp(double f) {
long seconds = (long) f;
@@ -576,6 +614,37 @@ public class TimestampWritable implements WritableComparable<TimestampWritable>
return t;
}
+ public static void setTimestampFromDouble(Timestamp timestamp, double f) {
+ // Otherwise, BigDecimal throws an exception. (Support vector operations that sometimes
+ // do work on double Not-a-Number NaN values).
+ if (Double.isNaN(f)) {
+ timestamp.setTime(0);
+ return;
+ }
+ // Algorithm used by TimestampWritable.doubleToTimestamp method.
+ // Allocates a BigDecimal object!
+
+ long seconds = (long) f;
+
+ // We must ensure the exactness of the double's fractional portion.
+ // 0.6 as the fraction part will be converted to 0.59999... and
+ // significantly reduce the savings from binary serialization
+ BigDecimal bd = new BigDecimal(String.valueOf(f));
+ bd = bd.subtract(new BigDecimal(seconds)).multiply(new BigDecimal(1000000000));
+ int nanos = bd.intValue();
+
+ // Convert to millis
+ long millis = seconds * 1000;
+ if (nanos < 0) {
+ millis -= 1000;
+ nanos += 1000000000;
+ }
+ timestamp.setTime(millis);
+
+ // Set remaining fractional portion to nanos
+ timestamp.setNanos(nanos);
+ }
+
public static void setTimestamp(Timestamp t, byte[] bytes, int offset) {
boolean hasDecimalOrSecondVInt = hasDecimalOrSecondVInt(bytes[offset]);
long seconds = (long) TimestampWritable.getSeconds(bytes, offset);
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/storage-api/src/java/org/apache/hadoop/hive/common/type/HiveIntervalDayTime.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/type/HiveIntervalDayTime.java b/storage-api/src/java/org/apache/hadoop/hive/common/type/HiveIntervalDayTime.java
new file mode 100644
index 0000000..b891e27
--- /dev/null
+++ b/storage-api/src/java/org/apache/hadoop/hive/common/type/HiveIntervalDayTime.java
@@ -0,0 +1,253 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.common.type;
+
+import java.math.BigDecimal;
+import java.sql.Timestamp;
+import java.util.Date;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.hive.common.util.IntervalDayTimeUtils;
+
+import sun.util.calendar.BaseCalendar;
+
+/**
+ * Day-time interval type representing an offset in days/hours/minutes/seconds,
+ * with nanosecond precision.
+ * 1 day = 24 hours = 1440 minutes = 86400 seconds
+ */
+public class HiveIntervalDayTime implements Comparable<HiveIntervalDayTime> {
+
+ // days/hours/minutes/seconds all represented as seconds
+ protected long totalSeconds;
+ protected int nanos;
+
+ public HiveIntervalDayTime() {
+ }
+
+ public HiveIntervalDayTime(int days, int hours, int minutes, int seconds, int nanos) {
+ set(days, hours, minutes, seconds, nanos);
+ }
+
+ public HiveIntervalDayTime(long seconds, int nanos) {
+ set(seconds, nanos);
+ }
+
+ public HiveIntervalDayTime(BigDecimal seconds) {
+ set(seconds);
+ }
+
+ public HiveIntervalDayTime(HiveIntervalDayTime other) {
+ set(other.totalSeconds, other.nanos);
+ }
+
+ public int getDays() {
+ return (int) TimeUnit.SECONDS.toDays(totalSeconds);
+ }
+
+ public int getHours() {
+ return (int) (TimeUnit.SECONDS.toHours(totalSeconds) % TimeUnit.DAYS.toHours(1));
+ }
+
+ public int getMinutes() {
+ return (int) (TimeUnit.SECONDS.toMinutes(totalSeconds) % TimeUnit.HOURS.toMinutes(1));
+ }
+
+ public int getSeconds() {
+ return (int) (totalSeconds % TimeUnit.MINUTES.toSeconds(1));
+ }
+
+ public int getNanos() {
+ return nanos;
+ }
+
+ /**
+ * Returns days/hours/minutes all converted into seconds.
+ * Nanos still need to be retrieved using getNanos()
+ * @return
+ */
+ public long getTotalSeconds() {
+ return totalSeconds;
+ }
+
+ /**
+ *
+ * @return double representation of the interval day time, accurate to nanoseconds
+ */
+ public double getDouble() {
+ return totalSeconds + nanos / 1000000000;
+ }
+
+ /**
+ * Ensures that the seconds and nanoseconds fields have consistent sign
+ */
+ protected void normalizeSecondsAndNanos() {
+ if (totalSeconds > 0 && nanos < 0) {
+ --totalSeconds;
+ nanos += IntervalDayTimeUtils.NANOS_PER_SEC;
+ } else if (totalSeconds < 0 && nanos > 0) {
+ ++totalSeconds;
+ nanos -= IntervalDayTimeUtils.NANOS_PER_SEC;
+ }
+ }
+
+ public void set(int days, int hours, int minutes, int seconds, int nanos) {
+ long totalSeconds = seconds;
+ totalSeconds += TimeUnit.DAYS.toSeconds(days);
+ totalSeconds += TimeUnit.HOURS.toSeconds(hours);
+ totalSeconds += TimeUnit.MINUTES.toSeconds(minutes);
+ totalSeconds += TimeUnit.NANOSECONDS.toSeconds(nanos);
+ nanos = nanos % IntervalDayTimeUtils.NANOS_PER_SEC;
+
+ this.totalSeconds = totalSeconds;
+ this.nanos = nanos;
+
+ normalizeSecondsAndNanos();
+ }
+
+ public void set(long seconds, int nanos) {
+ this.totalSeconds = seconds;
+ this.nanos = nanos;
+ normalizeSecondsAndNanos();
+ }
+
+ public void set(BigDecimal totalSecondsBd) {
+ long totalSeconds = totalSecondsBd.longValue();
+ BigDecimal fractionalSecs = totalSecondsBd.remainder(BigDecimal.ONE);
+ int nanos = fractionalSecs.multiply(IntervalDayTimeUtils.NANOS_PER_SEC_BD).intValue();
+ set(totalSeconds, nanos);
+ }
+
+ public void set(HiveIntervalDayTime other) {
+ set(other.getTotalSeconds(), other.getNanos());
+ }
+
+ public HiveIntervalDayTime negate() {
+ return new HiveIntervalDayTime(-getTotalSeconds(), -getNanos());
+ }
+
+ @Override
+ public int compareTo(HiveIntervalDayTime other) {
+ long cmp = this.totalSeconds - other.totalSeconds;
+ if (cmp == 0) {
+ cmp = this.nanos - other.nanos;
+ }
+ if (cmp != 0) {
+ cmp = cmp > 0 ? 1 : -1;
+ }
+ return (int) cmp;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof HiveIntervalDayTime)) {
+ return false;
+ }
+ return 0 == compareTo((HiveIntervalDayTime) obj);
+ }
+
+ /**
+ * Return a copy of this object.
+ */
+ public Object clone() {
+ return new HiveIntervalDayTime(totalSeconds, nanos);
+ }
+
+ @Override
+ public int hashCode() {
+ return new HashCodeBuilder().append(totalSeconds).append(nanos).toHashCode();
+ }
+
+ @Override
+ public String toString() {
+ // If normalize() was used, then day-hour-minute-second-nanos should have the same sign.
+ // This is currently working with that assumption.
+ boolean isNegative = (totalSeconds < 0 || nanos < 0);
+ String daySecondSignStr = isNegative ? "-" : "";
+
+ return String.format("%s%d %02d:%02d:%02d.%09d",
+ daySecondSignStr, Math.abs(getDays()),
+ Math.abs(getHours()), Math.abs(getMinutes()),
+ Math.abs(getSeconds()), Math.abs(getNanos()));
+ }
+
+ public static HiveIntervalDayTime valueOf(String strVal) {
+ HiveIntervalDayTime result = null;
+ if (strVal == null) {
+ throw new IllegalArgumentException("Interval day-time string was null");
+ }
+ Matcher patternMatcher = PATTERN_MATCHER.get();
+ patternMatcher.reset(strVal);
+ if (patternMatcher.matches()) {
+ // Parse out the individual parts
+ try {
+ // Sign - whether interval is positive or negative
+ int sign = 1;
+ String field = patternMatcher.group(1);
+ if (field != null && field.equals("-")) {
+ sign = -1;
+ }
+ int days = sign *
+ IntervalDayTimeUtils.parseNumericValueWithRange("day", patternMatcher.group(2),
+ 0, Integer.MAX_VALUE);
+ byte hours = (byte) (sign *
+ IntervalDayTimeUtils.parseNumericValueWithRange("hour", patternMatcher.group(3), 0, 23));
+ byte minutes = (byte) (sign *
+ IntervalDayTimeUtils.parseNumericValueWithRange("minute", patternMatcher.group(4), 0, 59));
+ int seconds = 0;
+ int nanos = 0;
+ field = patternMatcher.group(5);
+ if (field != null) {
+ BigDecimal bdSeconds = new BigDecimal(field);
+ if (bdSeconds.compareTo(IntervalDayTimeUtils.MAX_INT_BD) > 0) {
+ throw new IllegalArgumentException("seconds value of " + bdSeconds + " too large");
+ }
+ seconds = sign * bdSeconds.intValue();
+ nanos = sign * bdSeconds.subtract(new BigDecimal(bdSeconds.toBigInteger()))
+ .multiply(IntervalDayTimeUtils.NANOS_PER_SEC_BD).intValue();
+ }
+
+ result = new HiveIntervalDayTime(days, hours, minutes, seconds, nanos);
+ } catch (Exception err) {
+ throw new IllegalArgumentException("Error parsing interval day-time string: " + strVal, err);
+ }
+ } else {
+ throw new IllegalArgumentException(
+ "Interval string does not match day-time format of 'd h:m:s.n': " + strVal);
+ }
+
+ return result;
+ }
+
+ // Simple pattern: D H:M:S.nnnnnnnnn
+ private final static String PARSE_PATTERN =
+ "([+|-])?(\\d+) (\\d+):(\\d+):((\\d+)(\\.(\\d+))?)";
+
+ private static final ThreadLocal<Matcher> PATTERN_MATCHER = new ThreadLocal<Matcher>() {
+ @Override
+ protected Matcher initialValue() {
+ return Pattern.compile(PARSE_PATTERN).matcher("");
+ }
+ };
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/storage-api/src/java/org/apache/hadoop/hive/common/type/PisaTimestamp.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/type/PisaTimestamp.java b/storage-api/src/java/org/apache/hadoop/hive/common/type/PisaTimestamp.java
deleted file mode 100644
index ac1e38a..0000000
--- a/storage-api/src/java/org/apache/hadoop/hive/common/type/PisaTimestamp.java
+++ /dev/null
@@ -1,609 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.common.type;
-
-import java.math.BigDecimal;
-import java.sql.Timestamp;
-import java.util.Random;
-import java.util.concurrent.TimeUnit;
-
-import com.google.common.base.Preconditions;
-
-/**
- * Pisa project is named after the famous Leonardo of Pisa, or better known as Fibanacci.
- *
- * A Pisa timestamp is a timestamp without a time-zone (i.e. local) in the ISO-8601 calendar system,
- * such as 2007-12-03 10:15:30.0123456789, with accuracy to the nanosecond (1 billionth of a
- * second).
- *
- * Pisa timestamps use the same starting point as a java.sql.Timestamp -- the number of nanoseconds
- * since the epoch (1970-01-01, or the day Unix roared awake) where negative numbers represent
- * earlier days.
- *
- * However, we use the PisaTimestamp class which has different design requirements than
- * java.sql.Timestamp. It is designed to be mutable and NOT thread-safe to avoid high memory
- * allocation / garbage collection costs. And, provides for ease of use by our vectorization
- * code to avoid the high CPU data cache miss cost for small objects, too. We do this by allowing
- * the epoch day and nano of day to be stored externally (i.e. vector arrays).
- *
- * And, importantly, PisaTimestamp is a light-weight class similar to the epochDay/NanoOfDay of
- * the newer Java 8 LocalDateTime class, except the timestamp is *indifferent* to timezone.
- *
- * A common usage would be to treat it as UTC.
- *
- * You can work with days, seconds, milliseconds, nanoseconds, etc. But to work with months you
- * will need to convert to an external timestamp object and use calendars, etc.
- * *
- * The storage for a PisaTimestamp is:
- *
- * long epochDay
- * // The number of days since 1970-01-01 (==> similar to Java 8 LocalDate).
- * long nanoOfDay
- * // The number of nanoseconds within the day, with the range of
- * // 0 to 24 * 60 * 60 * 1,000,000,000 - 1 (==> similar to Java 8 LocalTime).
- *
- * Both epochDay and nanoOfDay are signed.
- *
- * We when both epochDay and nanoOfDay are non-zero, we will maintain them so they have the
- * same sign.
- *
- */
-
-public class PisaTimestamp {
-
- private static final long serialVersionUID = 1L;
-
- private long epochDay;
- private long nanoOfDay;
-
- private Timestamp scratchTimestamp;
-
- public static final long NANOSECONDS_PER_SECOND = TimeUnit.SECONDS.toNanos(1);
- public static final long NANOSECONDS_PER_MILLISECOND = TimeUnit.MILLISECONDS.toNanos(1);
- public static final long NANOSECONDS_PER_DAY = TimeUnit.DAYS.toNanos(1);
-
- public static final long MILLISECONDS_PER_SECOND = TimeUnit.SECONDS.toMillis(1);
- public static final long MILLISECONDS_PER_DAY = TimeUnit.DAYS.toMillis(1);
-
- public static final long SECONDS_PER_DAY = TimeUnit.DAYS.toSeconds(1);
-
- public static final long MIN_NANO_OF_DAY = -NANOSECONDS_PER_DAY;
- public static final long MAX_NANO_OF_DAY = NANOSECONDS_PER_DAY;
-
- public static final BigDecimal BIG_NANOSECONDS_PER_SECOND = new BigDecimal(NANOSECONDS_PER_SECOND);
-
- public long getEpochDay() {
- return epochDay;
- }
-
- public long getNanoOfDay() {
- return nanoOfDay;
- }
-
- public PisaTimestamp() {
- epochDay = 0;
- nanoOfDay = 0;
- scratchTimestamp = new Timestamp(0);
- }
-
- public PisaTimestamp(long epochDay, long nanoOfDay) {
-
- Preconditions.checkState(validateIntegrity(epochDay, nanoOfDay),
- "epochDay " + epochDay + ", nanoOfDay " + nanoOfDay + " not valid");
-
- this.epochDay = epochDay;
- this.nanoOfDay = nanoOfDay;
- scratchTimestamp = new Timestamp(0);
- }
-
- public PisaTimestamp(Timestamp timestamp) {
- super();
- updateFromTimestamp(timestamp);
- }
-
- public void reset() {
- epochDay = 0;
- nanoOfDay = 0;
- }
-
- /**
- * NOTE: This method validates the integrity rules between epoch day and nano of day,
- * but not overflow/underflow of epoch day. Since epoch day overflow/underflow can result
- * from to client data input, that must be checked manually with <undone> as this
- * class do not throw data range exceptions as a rule. It leaves that choice to the caller.
- * @param epochDay
- * @param nanoOfDay
- * @return true if epoch day and nano of day have integrity.
- */
- public static boolean validateIntegrity(long epochDay, long nanoOfDay) {
-
- // Range check nano per day as invariant.
- if (nanoOfDay >= NANOSECONDS_PER_DAY || nanoOfDay <= -NANOSECONDS_PER_DAY) {
- return false;
- }
-
- // Signs of epoch day and nano of day must match.
- if (!(epochDay >= 0 && nanoOfDay >= 0 ||
- epochDay <= 0 && nanoOfDay <= 0)) {
- return false;
- }
-
- return true;
- }
-
- /**
- * Set this PisaTimestamp from another PisaTimestamp.
- * @param source
- * @return this
- */
- public PisaTimestamp update(PisaTimestamp source) {
- this.epochDay = source.epochDay;
- this.nanoOfDay = source.nanoOfDay;
- return this;
- }
-
- /**
- * Set this PisaTimestamp from a epoch day and nano of day.
- * @param epochDay
- * @param nanoOfDay
- * @return this
- */
- public PisaTimestamp update(long epochDay, long nanoOfDay) {
-
- Preconditions.checkState(validateIntegrity(epochDay, nanoOfDay),
- "epochDay " + epochDay + ", nanoOfDay " + nanoOfDay + " not valid");
-
- this.epochDay = epochDay;
- this.nanoOfDay = nanoOfDay;
- return this;
- }
-
- /**
- * Set the PisaTimestamp from a Timestamp object.
- * @param timestamp
- * @return this
- */
- public PisaTimestamp updateFromTimestamp(Timestamp timestamp) {
-
- long timestampTime = timestamp.getTime();
- int nanos = timestamp.getNanos();
-
- /**
- * Since the Timestamp class always stores nanos as a positive quantity (0 .. 999,999,999),
- * we have to adjust back the time (subtract) by 1,000,000,000 to get right quantity for
- * our calculations below. One thing it ensures is nanoOfDay will be negative.
- */
- if (timestampTime < 0 && nanos > 0) {
- timestampTime -= MILLISECONDS_PER_SECOND;
- }
-
- // The Timestamp class does not use the milliseconds part (always 0). It is covered by nanos.
- long epochSeconds = timestampTime / MILLISECONDS_PER_SECOND;
-
- nanoOfDay = (epochSeconds % SECONDS_PER_DAY) * NANOSECONDS_PER_SECOND + nanos;
- epochDay = epochSeconds / SECONDS_PER_DAY + (nanoOfDay / NANOSECONDS_PER_DAY);
-
- Preconditions.checkState(validateIntegrity(epochDay, nanoOfDay));
- return this;
- }
-
- /**
- * Set this PisaTimestamp from a timestamp milliseconds.
- * @param epochMilliseconds
- * @return this
- */
- public PisaTimestamp updateFromTimestampMilliseconds(long timestampMilliseconds) {
- /**
- * The Timestamp class setTime sets both the time (seconds stored as milliseconds) and
- * the nanos.
- */
- scratchTimestamp.setTime(timestampMilliseconds);
- updateFromTimestamp(scratchTimestamp);
- return this;
- }
-
- /**
- * Set this PisaTimestamp from a timestamp seconds.
- * @param epochMilliseconds
- * @return this
- */
- public PisaTimestamp updateFromTimestampSeconds(long timestampSeconds) {
- return updateFromTimestampMilliseconds(timestampSeconds * MILLISECONDS_PER_SECOND);
- }
-
- /**
- * Set this PisaTimestamp from a timestamp seconds.
- * @param epochMilliseconds
- * @return this
- */
- public PisaTimestamp updateFromTimestampSecondsWithFractionalNanoseconds(
- double timestampSecondsWithFractionalNanoseconds) {
-
- // Otherwise, BigDecimal throws an exception. (Support vector operations that sometimes
- // do work on double Not-a-Number NaN values).
- if (Double.isNaN(timestampSecondsWithFractionalNanoseconds)) {
- timestampSecondsWithFractionalNanoseconds = 0;
- }
- // Algorithm used by TimestampWritable.doubleToTimestamp method.
- // Allocates a BigDecimal object!
-
- long seconds = (long) timestampSecondsWithFractionalNanoseconds;
-
- // We must ensure the exactness of the double's fractional portion.
- // 0.6 as the fraction part will be converted to 0.59999... and
- // significantly reduce the savings from binary serialization.
- BigDecimal bd;
-
- bd = new BigDecimal(String.valueOf(timestampSecondsWithFractionalNanoseconds));
- bd = bd.subtract(new BigDecimal(seconds)); // Get the nanos fraction.
- bd = bd.multiply(BIG_NANOSECONDS_PER_SECOND); // Make nanos an integer.
-
- int nanos = bd.intValue();
-
- // Convert to millis
- long millis = seconds * 1000;
- if (nanos < 0) {
- millis -= 1000;
- nanos += 1000000000;
- }
-
- scratchTimestamp.setTime(millis);
- scratchTimestamp.setNanos(nanos);
- updateFromTimestamp(scratchTimestamp);
- return this;
- }
-
- /**
- * Set this PisaTimestamp from a epoch seconds and signed nanos (-999999999 to 999999999).
- * @param epochSeconds
- * @param signedNanos
- * @return this
- */
- public PisaTimestamp updateFromEpochSecondsAndSignedNanos(long epochSeconds, int signedNanos) {
-
- long nanoOfDay = (epochSeconds % SECONDS_PER_DAY) * NANOSECONDS_PER_SECOND + signedNanos;
- long epochDay = epochSeconds / SECONDS_PER_DAY + nanoOfDay / NANOSECONDS_PER_DAY;
-
- Preconditions.checkState(validateIntegrity(epochDay, nanoOfDay));
-
- this.epochDay = epochDay;
- this.nanoOfDay = nanoOfDay;
- return this;
- }
-
- /**
- * Set a scratch PisaTimestamp with this PisaTimestamp's values and return the scratch object.
- * @param epochDay
- * @param nanoOfDay
- */
- public PisaTimestamp scratchCopy(PisaTimestamp scratch) {
-
- scratch.epochDay = epochDay;
- scratch.nanoOfDay = nanoOfDay;
- return scratch;
- }
-
- /**
- * Set a Timestamp object from this PisaTimestamp.
- * @param timestamp
- */
- public void timestampUpdate(Timestamp timestamp) {
-
- /*
- * java.sql.Timestamp consists of a long variable to store milliseconds and an integer variable for nanoseconds.
- * The long variable is used to store only the full seconds converted to millis. For example for 1234 milliseconds,
- * 1000 is stored in the long variable, and 234000000 (234 converted to nanoseconds) is stored as nanoseconds.
- * The negative timestamps are also supported, but nanoseconds must be positive therefore millisecond part is
- * reduced by one second.
- */
-
- long epochSeconds = epochDay * SECONDS_PER_DAY + nanoOfDay / NANOSECONDS_PER_SECOND;
- long integralSecInMillis;
- int nanos = (int) (nanoOfDay % NANOSECONDS_PER_SECOND); // The nanoseconds.
- if (nanos < 0) {
- nanos = (int) NANOSECONDS_PER_SECOND + nanos; // The positive nano-part that will be added to milliseconds.
- integralSecInMillis = (epochSeconds - 1) * MILLISECONDS_PER_SECOND; // Reduce by one second.
- } else {
- integralSecInMillis = epochSeconds * MILLISECONDS_PER_SECOND; // Full seconds converted to millis.
- }
-
- timestamp.setTime(integralSecInMillis);
- timestamp.setNanos(nanos);
- }
-
- /**
- * Return the scratch timestamp with values from Pisa timestamp.
- * @return
- */
- public Timestamp asScratchTimestamp() {
- timestampUpdate(scratchTimestamp);
- return scratchTimestamp;
- }
-
- /**
- * Return the scratch timestamp for use by the caller.
- * @return
- */
- public Timestamp useScratchTimestamp() {
- return scratchTimestamp;
- }
-
- public int compareTo(PisaTimestamp another) {
-
- if (epochDay == another.epochDay) {
- if (nanoOfDay == another.nanoOfDay){
- return 0;
- } else {
- return (nanoOfDay < another.nanoOfDay ? -1 : 1);
- }
- } else {
- return (epochDay < another.epochDay ? -1: 1);
- }
- }
-
- public static int compareTo(long epochDay1, long nanoOfDay1, PisaTimestamp another) {
-
- if (epochDay1 == another.epochDay) {
- if (nanoOfDay1 == another.nanoOfDay){
- return 0;
- } else {
- return (nanoOfDay1 < another.nanoOfDay ? -1 : 1);
- }
- } else {
- return (epochDay1 < another.epochDay ? -1: 1);
- }
- }
-
- public static int compareTo(PisaTimestamp pisaTimestamp1, long epochDay2, long nanoOfDay2) {
-
- if (pisaTimestamp1.epochDay == epochDay2) {
- if (pisaTimestamp1.nanoOfDay == nanoOfDay2){
- return 0;
- } else {
- return (pisaTimestamp1.nanoOfDay < nanoOfDay2 ? -1 : 1);
- }
- } else {
- return (pisaTimestamp1.epochDay < epochDay2 ? -1: 1);
- }
- }
-
- public static int compareTo(long epochDay1, long nanoOfDay1, long epochDay2, long nanoOfDay2) {
-
- if (epochDay1 == epochDay2) {
- if (nanoOfDay1 == nanoOfDay2){
- return 0;
- } else {
- return (nanoOfDay1 < nanoOfDay2 ? -1 : 1);
- }
- } else {
- return (epochDay1 < epochDay2 ? -1: 1);
- }
- }
-
-
- /**
- * Standard equals method override.
- */
- @Override
- public boolean equals(Object obj) {
- if (obj == null || obj.getClass() != getClass()) {
- return false;
- }
- return equals((PisaTimestamp) obj);
- }
-
- public boolean equals(PisaTimestamp other) {
-
- if (epochDay == other.epochDay) {
- if (nanoOfDay == other.nanoOfDay) {
- return true;
- } else {
- return false;
- }
- } else {
- return false;
- }
- }
-
- public static void add(PisaTimestamp pisaTimestamp1, PisaTimestamp pisaTimestamp2,
- PisaTimestamp result) {
- add(pisaTimestamp1.epochDay, pisaTimestamp1.nanoOfDay,
- pisaTimestamp2.epochDay, pisaTimestamp2.nanoOfDay,
- result);
- }
-
- public static void add(long epochDay1, long nanoOfDay1,
- long epochDay2, long nanoOfDay2,
- PisaTimestamp result) {
-
- // Validate integrity rules between epoch day and nano of day.
- Preconditions.checkState(PisaTimestamp.validateIntegrity(epochDay1, nanoOfDay1));
- Preconditions.checkState(PisaTimestamp.validateIntegrity(epochDay2, nanoOfDay2));
-
- long intermediateEpochDay = epochDay1 + epochDay2;
- long intermediateNanoOfDay = nanoOfDay1 + nanoOfDay2;
-
- // Normalize so both are positive or both are negative.
- long normalizedEpochDay;
- long normalizedNanoOfDay;
- if (intermediateEpochDay > 0 && intermediateNanoOfDay < 0) {
- normalizedEpochDay = intermediateEpochDay - 1;
- normalizedNanoOfDay = intermediateNanoOfDay + NANOSECONDS_PER_DAY;
- } else if (intermediateEpochDay < 0 && intermediateNanoOfDay > 0) {
- normalizedEpochDay = intermediateEpochDay + 1;
- normalizedNanoOfDay = intermediateNanoOfDay - NANOSECONDS_PER_DAY;
- } else {
- normalizedEpochDay = intermediateEpochDay;
- normalizedNanoOfDay = intermediateNanoOfDay;
- }
-
- long resultEpochDay;
- long resultNanoOfDay;
- if (normalizedNanoOfDay >= NANOSECONDS_PER_DAY || normalizedNanoOfDay <= -NANOSECONDS_PER_DAY) {
- // Adjust for carry or overflow...
-
- resultEpochDay = normalizedEpochDay + normalizedNanoOfDay / NANOSECONDS_PER_DAY;
- resultNanoOfDay = normalizedNanoOfDay % NANOSECONDS_PER_DAY;
-
- } else {
- resultEpochDay = normalizedEpochDay;
- resultNanoOfDay = normalizedNanoOfDay;
- }
-
- // The update method will validate integrity rules between epoch day and nano of day,
- // but not overflow/underflow of epoch day.
- result.update(resultEpochDay, resultNanoOfDay);
- }
-
- public static void addSeconds(PisaTimestamp timestamp1, long epochSeconds, PisaTimestamp result) {
- long epochDay = epochSeconds / SECONDS_PER_DAY;
- long nanoOfDay = (epochSeconds % SECONDS_PER_DAY) * NANOSECONDS_PER_SECOND;
- add(timestamp1.epochDay, timestamp1.nanoOfDay, epochDay, nanoOfDay, result);
- }
-
- public static void subtract(PisaTimestamp timestamp1, PisaTimestamp timestamp2,
- PisaTimestamp result) {
-
- add(timestamp1.epochDay, timestamp1.nanoOfDay, -timestamp2.epochDay, -timestamp2.nanoOfDay,
- result);
- }
-
- public static void subtract(long epochDay1, long nanoOfDay1,
- long epochDay2, long nanoOfDay2,
- PisaTimestamp result) {
-
- add(epochDay1, nanoOfDay1, -epochDay2, -nanoOfDay2, result);
- }
-
- public static void subtractSeconds(PisaTimestamp timestamp1, long epochSeconds,
- PisaTimestamp result) {
- long epochDay = epochSeconds / SECONDS_PER_DAY;
- long nanoOfDay = (epochSeconds % SECONDS_PER_DAY) * NANOSECONDS_PER_SECOND;
- add(timestamp1.epochDay, timestamp1.nanoOfDay, -epochDay, -nanoOfDay, result);
- }
-
- /**
- * Rounds the number of milliseconds relative to the epoch down to the nearest whole number of
- * seconds. 500 would round to 0, -500 would round to -1.
- */
- public static long timestampMillisToSeconds(long millis) {
- if (millis >= 0) {
- return millis / 1000;
- } else {
- return (millis - 999) / 1000;
- }
- }
-
- /**
- * Return a double with the integer part as the seconds and the fractional part as
- * the nanoseconds the way the Timestamp class does it.
- * @return seconds.nanoseconds
- */
- public double getTimestampSecondsWithFractionalNanos() {
- // Algorithm must be the same as TimestampWritable.getDouble method.
- timestampUpdate(scratchTimestamp);
- double seconds = timestampMillisToSeconds(scratchTimestamp.getTime());
- double nanos = scratchTimestamp.getNanos();
- BigDecimal bigSeconds = new BigDecimal(seconds);
- BigDecimal bigNanos = new BigDecimal(nanos).divide(BIG_NANOSECONDS_PER_SECOND);
- return bigSeconds.add(bigNanos).doubleValue();
- }
-
- /**
- * Return an integer as the seconds the way the Timestamp class does it.
- * @return seconds.nanoseconds
- */
- public long getTimestampSeconds() {
- // Algorithm must be the same as TimestampWritable.getSeconds method.
- timestampUpdate(scratchTimestamp);
- return timestampMillisToSeconds(scratchTimestamp.getTime());
- }
-
- /**
- * Return an integer as the milliseconds the way the Timestamp class does it.
- * @return seconds.nanoseconds
- */
- public long getTimestampMilliseconds() {
- timestampUpdate(scratchTimestamp);
- return scratchTimestamp.getTime();
- }
-
- /**
- * Return the epoch seconds.
- * @return
- */
- public long getEpochSeconds() {
- return epochDay * SECONDS_PER_DAY + nanoOfDay / NANOSECONDS_PER_SECOND;
- }
-
- /**
- * Return the epoch seconds, given the epoch day and nano of day.
- * @param epochDay
- * @param nanoOfDay
- * @return
- */
- public static long getEpochSecondsFromEpochDayAndNanoOfDay(long epochDay, long nanoOfDay) {
- return epochDay * SECONDS_PER_DAY + nanoOfDay / NANOSECONDS_PER_SECOND;
- }
-
- /**
- * Return the signed nanos (-999999999 to 999999999).
- * NOTE: Not the same as Timestamp class nanos (which are always positive).
- */
- public int getSignedNanos() {
- return (int) (nanoOfDay % NANOSECONDS_PER_SECOND);
- }
-
- /**
- * Return the signed nanos (-999999999 to 999999999).
- * NOTE: Not the same as Timestamp class nanos (which are always positive).
- */
- public static int getSignedNanos(long nanoOfDay) {
- return (int) (nanoOfDay % NANOSECONDS_PER_SECOND);
- }
-
- /**
- * Return the epoch milliseconds.
- * @return
- */
- public long getEpochMilliseconds() {
- return epochDay * MILLISECONDS_PER_DAY + nanoOfDay / NANOSECONDS_PER_MILLISECOND;
- }
-
- /**
- * Return the epoch seconds, given the epoch day and nano of day.
- * @param epochDay
- * @param nanoOfDay
- * @return
- */
- public static long getEpochMillisecondsFromEpochDayAndNanoOfDay(long epochDay, long nanoOfDay) {
- return epochDay * MILLISECONDS_PER_DAY + nanoOfDay / NANOSECONDS_PER_MILLISECOND;
- }
-
- @Override
- public int hashCode() {
- // UNDONE: We don't want to box the longs just to get the hash codes...
- return new Long(epochDay).hashCode() ^ new Long(nanoOfDay).hashCode();
- }
-
- @Override
- public String toString() {
- timestampUpdate(scratchTimestamp);
- return scratchTimestamp.toString();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/storage-api/src/java/org/apache/hadoop/hive/common/type/RandomTypeUtil.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/type/RandomTypeUtil.java b/storage-api/src/java/org/apache/hadoop/hive/common/type/RandomTypeUtil.java
index 13baff4..3fb0cfd 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/common/type/RandomTypeUtil.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/common/type/RandomTypeUtil.java
@@ -18,21 +18,67 @@
package org.apache.hadoop.hive.common.type;
import java.sql.Timestamp;
+import java.text.DateFormat;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
import java.util.Random;
import java.util.concurrent.TimeUnit;
public class RandomTypeUtil {
public static final long NANOSECONDS_PER_SECOND = TimeUnit.SECONDS.toNanos(1);
+ public static final long MILLISECONDS_PER_SECOND = TimeUnit.SECONDS.toMillis(1);
+ public static final long NANOSECONDS_PER_MILLISSECOND = TimeUnit.MILLISECONDS.toNanos(1);
+
+ private static ThreadLocal<DateFormat> DATE_FORMAT =
+ new ThreadLocal<DateFormat>() {
+ @Override
+ protected DateFormat initialValue() {
+ return new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ }
+ };
+
+ // We've switched to Joda/Java Calendar which has a more limited time range....
+ public static int MIN_YEAR = 1900;
+ public static int MAX_YEAR = 3000;
+ private static long MIN_FOUR_DIGIT_YEAR_MILLIS = parseToMillis("1900-01-01 00:00:00");
+ private static long MAX_FOUR_DIGIT_YEAR_MILLIS = parseToMillis("3000-01-01 00:00:00");
+
+ private static long parseToMillis(String s) {
+ try {
+ return DATE_FORMAT.get().parse(s).getTime();
+ } catch (ParseException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
public static Timestamp getRandTimestamp(Random r) {
+ return getRandTimestamp(r, MIN_YEAR, MAX_YEAR);
+ }
+
+ public static Timestamp getRandTimestamp(Random r, int minYear, int maxYear) {
String optionalNanos = "";
- if (r.nextInt(2) == 1) {
+ switch (r.nextInt(4)) {
+ case 0:
+ // No nanos.
+ break;
+ case 1:
+ optionalNanos = String.format(".%09d",
+ Integer.valueOf(r.nextInt((int) NANOSECONDS_PER_SECOND)));
+ break;
+ case 2:
+ // Limit to milliseconds only...
optionalNanos = String.format(".%09d",
- Integer.valueOf(0 + r.nextInt((int) NANOSECONDS_PER_SECOND)));
+ Integer.valueOf(r.nextInt((int) MILLISECONDS_PER_SECOND)) * NANOSECONDS_PER_MILLISSECOND);
+ break;
+ case 3:
+ // Limit to below milliseconds only...
+ optionalNanos = String.format(".%09d",
+ Integer.valueOf(r.nextInt((int) NANOSECONDS_PER_MILLISSECOND)));
+ break;
}
String timestampStr = String.format("%04d-%02d-%02d %02d:%02d:%02d%s",
- Integer.valueOf(0 + r.nextInt(10000)), // year
+ Integer.valueOf(minYear + r.nextInt(maxYear - minYear + 1)), // year
Integer.valueOf(1 + r.nextInt(12)), // month
Integer.valueOf(1 + r.nextInt(28)), // day
Integer.valueOf(0 + r.nextInt(24)), // hour
@@ -48,4 +94,22 @@ public class RandomTypeUtil {
}
return timestampVal;
}
+
+ public static long randomMillis(long minMillis, long maxMillis, Random rand) {
+ return minMillis + (long) ((maxMillis - minMillis) * rand.nextDouble());
+ }
+
+ public static long randomMillis(Random rand) {
+ return randomMillis(MIN_FOUR_DIGIT_YEAR_MILLIS, MAX_FOUR_DIGIT_YEAR_MILLIS, rand);
+ }
+
+ public static int randomNanos(Random rand, int decimalDigits) {
+ // Only keep the most significant decimalDigits digits.
+ int nanos = rand.nextInt((int) NANOSECONDS_PER_SECOND);
+ return nanos - nanos % (int) Math.pow(10, 9 - decimalDigits);
+ }
+
+ public static int randomNanos(Random rand) {
+ return randomNanos(rand, 9);
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java
index 4ae9c47..c069a5f 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hive.ql.exec.vector;
-import java.io.IOException;
import java.util.Arrays;
/**
@@ -43,6 +42,7 @@ public abstract class ColumnVector {
BYTES,
DECIMAL,
TIMESTAMP,
+ INTERVAL_DAY_TIME,
STRUCT,
LIST,
MAP,
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/IntervalDayTimeColumnVector.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/IntervalDayTimeColumnVector.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/IntervalDayTimeColumnVector.java
new file mode 100644
index 0000000..39ccea8
--- /dev/null
+++ b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/IntervalDayTimeColumnVector.java
@@ -0,0 +1,348 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.exec.vector;
+
+import java.util.Arrays;
+
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
+import org.apache.hadoop.io.Writable;
+
+/**
+ * This class represents a nullable interval day time column vector capable of handing a
+ * wide range of interval day time values.
+ *
+ * We store the 2 (value) fields of a HiveIntervalDayTime class in primitive arrays.
+ *
+ * We do this to avoid an array of Java HiveIntervalDayTime objects which would have poor storage
+ * and memory access characteristics.
+ *
+ * Generally, the caller will fill in a scratch HiveIntervalDayTime object with values from a row,
+ * work using the scratch HiveIntervalDayTime, and then perhaps update the column vector row
+ * with a result.
+ */
+public class IntervalDayTimeColumnVector extends ColumnVector {
+
+ /*
+ * The storage arrays for this column vector corresponds to the storage of a HiveIntervalDayTime:
+ */
+ private long[] totalSeconds;
+ // The values from HiveIntervalDayTime.getTotalSeconds().
+
+ private int[] nanos;
+ // The values from HiveIntervalDayTime.getNanos().
+
+ /*
+ * Scratch objects.
+ */
+ private final HiveIntervalDayTime scratchIntervalDayTime;
+
+ private Writable scratchWritable;
+ // Supports keeping a HiveIntervalDayTimeWritable object without having to import
+ // that definition...
+
+ /**
+ * Use this constructor by default. All column vectors
+ * should normally be the default size.
+ */
+ public IntervalDayTimeColumnVector() {
+ this(VectorizedRowBatch.DEFAULT_SIZE);
+ }
+
+ /**
+ * Don't use this except for testing purposes.
+ *
+ * @param len the number of rows
+ */
+ public IntervalDayTimeColumnVector(int len) {
+ super(len);
+
+ totalSeconds = new long[len];
+ nanos = new int[len];
+
+ scratchIntervalDayTime = new HiveIntervalDayTime();
+
+ scratchWritable = null; // Allocated by caller.
+ }
+
+ /**
+ * Return the number of rows.
+ * @return
+ */
+ public int getLength() {
+ return totalSeconds.length;
+ }
+
+ /**
+ * Return a row's HiveIntervalDayTime.getTotalSeconds() value.
+ * We assume the entry has already been NULL checked and isRepeated adjusted.
+ * @param elementNum
+ * @return
+ */
+ public long getTotalSeconds(int elementNum) {
+ return totalSeconds[elementNum];
+ }
+
+ /**
+ * Return a row's HiveIntervalDayTime.getNanos() value.
+ * We assume the entry has already been NULL checked and isRepeated adjusted.
+ * @param elementNum
+ * @return
+ */
+ public long getNanos(int elementNum) {
+ return nanos[elementNum];
+ }
+
+ /**
+ * Return a row's HiveIntervalDayTime.getDouble() value.
+ * We assume the entry has already been NULL checked and isRepeated adjusted.
+ * @param elementNum
+ * @return
+ */
+ public double getDouble(int elementNum) {
+ return asScratchIntervalDayTime(elementNum).getDouble();
+ }
+
+ /**
+ * Set a HiveIntervalDayTime object from a row of the column.
+ * We assume the entry has already been NULL checked and isRepeated adjusted.
+ * @param intervalDayTime
+ * @param elementNum
+ */
+ public void intervalDayTimeUpdate(HiveIntervalDayTime intervalDayTime, int elementNum) {
+ intervalDayTime.set(totalSeconds[elementNum], nanos[elementNum]);
+ }
+
+
+ /**
+ * Return the scratch HiveIntervalDayTime object set from a row.
+ * We assume the entry has already been NULL checked and isRepeated adjusted.
+ * @param elementNum
+ * @return
+ */
+ public HiveIntervalDayTime asScratchIntervalDayTime(int elementNum) {
+ scratchIntervalDayTime.set(totalSeconds[elementNum], nanos[elementNum]);
+ return scratchIntervalDayTime;
+ }
+
+ /**
+ * Return the scratch HiveIntervalDayTime (contents undefined).
+ * @return
+ */
+ public HiveIntervalDayTime getScratchIntervalDayTime() {
+ return scratchIntervalDayTime;
+ }
+
+ /**
+ * Compare row to HiveIntervalDayTime.
+ * We assume the entry has already been NULL checked and isRepeated adjusted.
+ * @param elementNum
+ * @param intervalDayTime
+ * @return -1, 0, 1 standard compareTo values.
+ */
+ public int compareTo(int elementNum, HiveIntervalDayTime intervalDayTime) {
+ return asScratchIntervalDayTime(elementNum).compareTo(intervalDayTime);
+ }
+
+ /**
+ * Compare HiveIntervalDayTime to row.
+ * We assume the entry has already been NULL checked and isRepeated adjusted.
+ * @param intervalDayTime
+ * @param elementNum
+ * @return -1, 0, 1 standard compareTo values.
+ */
+ public int compareTo(HiveIntervalDayTime intervalDayTime, int elementNum) {
+ return intervalDayTime.compareTo(asScratchIntervalDayTime(elementNum));
+ }
+
+ /**
+ * Compare a row to another TimestampColumnVector's row.
+ * @param elementNum1
+ * @param intervalDayTimeColVector2
+ * @param elementNum2
+ * @return
+ */
+ public int compareTo(int elementNum1, IntervalDayTimeColumnVector intervalDayTimeColVector2,
+ int elementNum2) {
+ return asScratchIntervalDayTime(elementNum1).compareTo(
+ intervalDayTimeColVector2.asScratchIntervalDayTime(elementNum2));
+ }
+
+ /**
+ * Compare another TimestampColumnVector's row to a row.
+ * @param intervalDayTimeColVector1
+ * @param elementNum1
+ * @param elementNum2
+ * @return
+ */
+ public int compareTo(IntervalDayTimeColumnVector intervalDayTimeColVector1, int elementNum1,
+ int elementNum2) {
+ return intervalDayTimeColVector1.asScratchIntervalDayTime(elementNum1).compareTo(
+ asScratchIntervalDayTime(elementNum2));
+ }
+
+ @Override
+ public void setElement(int outElementNum, int inputElementNum, ColumnVector inputVector) {
+
+ IntervalDayTimeColumnVector timestampColVector = (IntervalDayTimeColumnVector) inputVector;
+
+ totalSeconds[outElementNum] = timestampColVector.totalSeconds[inputElementNum];
+ nanos[outElementNum] = timestampColVector.nanos[inputElementNum];
+ }
+
+ // Simplify vector by brute-force flattening noNulls and isRepeating
+ // This can be used to reduce combinatorial explosion of code paths in VectorExpressions
+ // with many arguments.
+ public void flatten(boolean selectedInUse, int[] sel, int size) {
+ flattenPush();
+ if (isRepeating) {
+ isRepeating = false;
+ long repeatFastTime = totalSeconds[0];
+ int repeatNanos = nanos[0];
+ if (selectedInUse) {
+ for (int j = 0; j < size; j++) {
+ int i = sel[j];
+ totalSeconds[i] = repeatFastTime;
+ nanos[i] = repeatNanos;
+ }
+ } else {
+ Arrays.fill(totalSeconds, 0, size, repeatFastTime);
+ Arrays.fill(nanos, 0, size, repeatNanos);
+ }
+ flattenRepeatingNulls(selectedInUse, sel, size);
+ }
+ flattenNoNulls(selectedInUse, sel, size);
+ }
+
+ /**
+ * Set a row from a HiveIntervalDayTime.
+ * We assume the entry has already been isRepeated adjusted.
+ * @param elementNum
+ * @param intervalDayTime
+ */
+ public void set(int elementNum, HiveIntervalDayTime intervalDayTime) {
+ this.totalSeconds[elementNum] = intervalDayTime.getTotalSeconds();
+ this.nanos[elementNum] = intervalDayTime.getNanos();
+ }
+
+ /**
+ * Set a row from the current value in the scratch interval day time.
+ * @param elementNum
+ */
+ public void setFromScratchIntervalDayTime(int elementNum) {
+ this.totalSeconds[elementNum] = scratchIntervalDayTime.getTotalSeconds();
+ this.nanos[elementNum] = scratchIntervalDayTime.getNanos();
+ }
+
+ /**
+ * Set row to standard null value(s).
+ * We assume the entry has already been isRepeated adjusted.
+ * @param elementNum
+ */
+ public void setNullValue(int elementNum) {
+ totalSeconds[elementNum] = 0;
+ nanos[elementNum] = 1;
+ }
+
+ // Copy the current object contents into the output. Only copy selected entries,
+ // as indicated by selectedInUse and the sel array.
+ public void copySelected(
+ boolean selectedInUse, int[] sel, int size, IntervalDayTimeColumnVector output) {
+
+ // Output has nulls if and only if input has nulls.
+ output.noNulls = noNulls;
+ output.isRepeating = false;
+
+ // Handle repeating case
+ if (isRepeating) {
+ output.totalSeconds[0] = totalSeconds[0];
+ output.nanos[0] = nanos[0];
+ output.isNull[0] = isNull[0];
+ output.isRepeating = true;
+ return;
+ }
+
+ // Handle normal case
+
+ // Copy data values over
+ if (selectedInUse) {
+ for (int j = 0; j < size; j++) {
+ int i = sel[j];
+ output.totalSeconds[i] = totalSeconds[i];
+ output.nanos[i] = nanos[i];
+ }
+ }
+ else {
+ System.arraycopy(totalSeconds, 0, output.totalSeconds, 0, size);
+ System.arraycopy(nanos, 0, output.nanos, 0, size);
+ }
+
+ // Copy nulls over if needed
+ if (!noNulls) {
+ if (selectedInUse) {
+ for (int j = 0; j < size; j++) {
+ int i = sel[j];
+ output.isNull[i] = isNull[i];
+ }
+ }
+ else {
+ System.arraycopy(isNull, 0, output.isNull, 0, size);
+ }
+ }
+ }
+
+ /**
+ * Fill all the vector entries with a HiveIntervalDayTime.
+ * @param intervalDayTime
+ */
+ public void fill(HiveIntervalDayTime intervalDayTime) {
+ noNulls = true;
+ isRepeating = true;
+ totalSeconds[0] = intervalDayTime.getTotalSeconds();
+ nanos[0] = intervalDayTime.getNanos();
+ }
+
+ /**
+ * Return a convenience writable object stored by this column vector.
+ * Supports keeping a TimestampWritable object without having to import that definition...
+ * @return
+ */
+ public Writable getScratchWritable() {
+ return scratchWritable;
+ }
+
+ /**
+ * Set the convenience writable object stored by this column vector
+ * @param scratchWritable
+ */
+ public void setScratchWritable(Writable scratchWritable) {
+ this.scratchWritable = scratchWritable;
+ }
+
+ @Override
+ public void stringifyValue(StringBuilder buffer, int row) {
+ if (isRepeating) {
+ row = 0;
+ }
+ if (noNulls || !isNull[row]) {
+ scratchIntervalDayTime.set(totalSeconds[row], nanos[row]);
+ buffer.append(scratchIntervalDayTime.toString());
+ } else {
+ buffer.append("null");
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java
index b73a0d2..c0dd5ed 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java
@@ -20,37 +20,35 @@ package org.apache.hadoop.hive.ql.exec.vector;
import java.sql.Timestamp;
import java.util.Arrays;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
-import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.io.Writable;
-import com.google.common.base.Preconditions;
-
/**
* This class represents a nullable timestamp column vector capable of handing a wide range of
* timestamp values.
*
- * We use the PisaTimestamp which is designed to be mutable and avoid the heavy memory allocation
- * and CPU data cache miss costs.
+ * We store the 2 (value) fields of a Timestamp class in primitive arrays.
+ *
+ * We do this to avoid an array of Java Timestamp objects which would have poor storage
+ * and memory access characteristics.
+ *
+ * Generally, the caller will fill in a scratch timestamp object with values from a row, work
+ * using the scratch timestamp, and then perhaps update the column vector row with a result.
*/
public class TimestampColumnVector extends ColumnVector {
/*
- * The storage arrays for this column vector corresponds to the storage of a PisaTimestamp:
+ * The storage arrays for this column vector corresponds to the storage of a Timestamp:
*/
- private long[] epochDay;
- // An array of the number of days since 1970-01-01 (similar to Java 8 LocalDate).
+ public long[] time;
+ // The values from Timestamp.getTime().
- private long[] nanoOfDay;
- // An array of the number of nanoseconds within the day, with the range of
- // 0 to 24 * 60 * 60 * 1,000,000,000 - 1 (similar to Java 8 LocalTime).
+ public int[] nanos;
+ // The values from Timestamp.getNanos().
/*
* Scratch objects.
*/
- private PisaTimestamp scratchPisaTimestamp;
- // Convenience scratch Pisa timestamp object.
+ private final Timestamp scratchTimestamp;
private Writable scratchWritable;
// Supports keeping a TimestampWritable object without having to import that definition...
@@ -71,10 +69,10 @@ public class TimestampColumnVector extends ColumnVector {
public TimestampColumnVector(int len) {
super(len);
- epochDay = new long[len];
- nanoOfDay = new long[len];
+ time = new long[len];
+ nanos = new int[len];
- scratchPisaTimestamp = new PisaTimestamp();
+ scratchTimestamp = new Timestamp(0);
scratchWritable = null; // Allocated by caller.
}
@@ -84,48 +82,27 @@ public class TimestampColumnVector extends ColumnVector {
* @return
*/
public int getLength() {
- return epochDay.length;
+ return time.length;
}
/**
- * Returnt a row's epoch day.
+ * Return a row's Timestamp.getTime() value.
* We assume the entry has already been NULL checked and isRepeated adjusted.
* @param elementNum
* @return
*/
- public long getEpochDay(int elementNum) {
- return epochDay[elementNum];
+ public long getTime(int elementNum) {
+ return time[elementNum];
}
/**
- * Return a row's nano of day.
+ * Return a row's Timestamp.getNanos() value.
* We assume the entry has already been NULL checked and isRepeated adjusted.
* @param elementNum
* @return
*/
- public long getNanoOfDay(int elementNum) {
- return nanoOfDay[elementNum];
- }
-
- /**
- * Get a scratch PisaTimestamp object from a row of the column.
- * We assume the entry has already been NULL checked and isRepeated adjusted.
- * @param elementNum
- * @return scratch
- */
- public PisaTimestamp asScratchPisaTimestamp(int elementNum) {
- scratchPisaTimestamp.update(epochDay[elementNum], nanoOfDay[elementNum]);
- return scratchPisaTimestamp;
- }
-
- /**
- * Set a PisaTimestamp object from a row of the column.
- * We assume the entry has already been NULL checked and isRepeated adjusted.
- * @param pisaTimestamp
- * @param elementNum
- */
- public void pisaTimestampUpdate(PisaTimestamp pisaTimestamp, int elementNum) {
- pisaTimestamp.update(epochDay[elementNum], nanoOfDay[elementNum]);
+ public int getNanos(int elementNum) {
+ return nanos[elementNum];
}
/**
@@ -135,154 +112,133 @@ public class TimestampColumnVector extends ColumnVector {
* @param elementNum
*/
public void timestampUpdate(Timestamp timestamp, int elementNum) {
- scratchPisaTimestamp.update(epochDay[elementNum], nanoOfDay[elementNum]);
- scratchPisaTimestamp.timestampUpdate(timestamp);
+ timestamp.setTime(time[elementNum]);
+ timestamp.setNanos(nanos[elementNum]);
}
/**
- * Compare row to PisaTimestamp.
+ * Return the scratch Timestamp object set from a row.
* We assume the entry has already been NULL checked and isRepeated adjusted.
* @param elementNum
- * @param pisaTimestamp
- * @return -1, 0, 1 standard compareTo values.
- */
- public int compareTo(int elementNum, PisaTimestamp pisaTimestamp) {
- return PisaTimestamp.compareTo(epochDay[elementNum], nanoOfDay[elementNum], pisaTimestamp);
- }
-
- /**
- * Compare PisaTimestamp to row.
- * We assume the entry has already been NULL checked and isRepeated adjusted.
- * @param pisaTimestamp
- * @param elementNum
- * @return -1, 0, 1 standard compareTo values.
+ * @return
*/
- public int compareTo(PisaTimestamp pisaTimestamp, int elementNum) {
- return PisaTimestamp.compareTo(pisaTimestamp, epochDay[elementNum], nanoOfDay[elementNum]);
+ public Timestamp asScratchTimestamp(int elementNum) {
+ scratchTimestamp.setTime(time[elementNum]);
+ scratchTimestamp.setNanos(nanos[elementNum]);
+ return scratchTimestamp;
}
/**
- * Compare a row to another TimestampColumnVector's row.
- * @param elementNum1
- * @param timestampColVector2
- * @param elementNum2
+ * Return the scratch timestamp (contents undefined).
* @return
*/
- public int compareTo(int elementNum1, TimestampColumnVector timestampColVector2,
- int elementNum2) {
- return PisaTimestamp.compareTo(
- epochDay[elementNum1], nanoOfDay[elementNum1],
- timestampColVector2.epochDay[elementNum2], timestampColVector2.nanoOfDay[elementNum2]);
+ public Timestamp getScratchTimestamp() {
+ return scratchTimestamp;
}
/**
- * Compare another TimestampColumnVector's row to a row.
- * @param timestampColVector1
- * @param elementNum1
- * @param elementNum2
+ * Return a long representation of a Timestamp.
+ * @param elementNum
* @return
*/
- public int compareTo(TimestampColumnVector timestampColVector1, int elementNum1,
- int elementNum2) {
- return PisaTimestamp.compareTo(
- timestampColVector1.epochDay[elementNum1], timestampColVector1.nanoOfDay[elementNum1],
- epochDay[elementNum2], nanoOfDay[elementNum2]);
- }
-
- public void add(PisaTimestamp timestamp1, PisaTimestamp timestamp2, int resultElementNum) {
- PisaTimestamp.add(timestamp1, timestamp2, scratchPisaTimestamp);
- epochDay[resultElementNum] = scratchPisaTimestamp.getEpochDay();
- nanoOfDay[resultElementNum] = scratchPisaTimestamp.getNanoOfDay();
- }
-
- public void subtract(PisaTimestamp timestamp1, PisaTimestamp timestamp2, int resultElementNum) {
- PisaTimestamp.subtract(timestamp1, timestamp2, scratchPisaTimestamp);
- epochDay[resultElementNum] = scratchPisaTimestamp.getEpochDay();
- nanoOfDay[resultElementNum] = scratchPisaTimestamp.getNanoOfDay();
+ public long getTimestampAsLong(int elementNum) {
+ scratchTimestamp.setTime(time[elementNum]);
+ scratchTimestamp.setNanos(nanos[elementNum]);
+ return getTimestampAsLong(scratchTimestamp);
}
/**
- * Return row as a double with the integer part as the seconds and the fractional part as
- * the nanoseconds the way the Timestamp class does it.
- * We assume the entry has already been NULL checked and isRepeated adjusted.
- * @param elementNum
- * @return seconds.nanoseconds
+ * Return a long representation of a Timestamp.
+ * @param timestamp
+ * @return
*/
- public double getTimestampSecondsWithFractionalNanos(int elementNum) {
- scratchPisaTimestamp.update(epochDay[elementNum], nanoOfDay[elementNum]);
- return scratchPisaTimestamp.getTimestampSecondsWithFractionalNanos();
+ public static long getTimestampAsLong(Timestamp timestamp) {
+ return millisToSeconds(timestamp.getTime());
}
+ // Copy of TimestampWritable.millisToSeconds
/**
- * Return row as integer as the seconds the way the Timestamp class does it.
- * We assume the entry has already been NULL checked and isRepeated adjusted.
- * @param elementNum
- * @return seconds
+ * Rounds the number of milliseconds relative to the epoch down to the nearest whole number of
+ * seconds. 500 would round to 0, -500 would round to -1.
*/
- public long getTimestampSeconds(int elementNum) {
- scratchPisaTimestamp.update(epochDay[elementNum], nanoOfDay[elementNum]);
- return scratchPisaTimestamp.getTimestampSeconds();
+ private static long millisToSeconds(long millis) {
+ if (millis >= 0) {
+ return millis / 1000;
+ } else {
+ return (millis - 999) / 1000;
+ }
}
-
/**
- * Return row as milliseconds the way the Timestamp class does it.
- * We assume the entry has already been NULL checked and isRepeated adjusted.
+ * Return a double representation of a Timestamp.
* @param elementNum
* @return
*/
- public long getTimestampMilliseconds(int elementNum) {
- scratchPisaTimestamp.update(epochDay[elementNum], nanoOfDay[elementNum]);
- return scratchPisaTimestamp.getTimestampMilliseconds();
+ public double getDouble(int elementNum) {
+ scratchTimestamp.setTime(time[elementNum]);
+ scratchTimestamp.setNanos(nanos[elementNum]);
+ return getDouble(scratchTimestamp);
}
/**
- * Return row as epoch seconds.
- * We assume the entry has already been NULL checked and isRepeated adjusted.
+ * Return a double representation of a Timestamp.
* @param elementNum
* @return
*/
- public long getEpochSeconds(int elementNum) {
- return PisaTimestamp.getEpochSecondsFromEpochDayAndNanoOfDay(epochDay[elementNum], nanoOfDay[elementNum]);
+ public static double getDouble(Timestamp timestamp) {
+ // Same algorithm as TimestampWritable (not currently import-able here).
+ double seconds, nanos;
+ seconds = millisToSeconds(timestamp.getTime());
+ nanos = timestamp.getNanos();
+ return seconds + nanos / 1000000000;
}
/**
- * Return row as epoch milliseconds.
+ * Compare row to Timestamp.
* We assume the entry has already been NULL checked and isRepeated adjusted.
* @param elementNum
- * @return
+ * @param timestamp
+ * @return -1, 0, 1 standard compareTo values.
*/
- public long getEpochMilliseconds(int elementNum) {
- return PisaTimestamp.getEpochMillisecondsFromEpochDayAndNanoOfDay(epochDay[elementNum], nanoOfDay[elementNum]);
+ public int compareTo(int elementNum, Timestamp timestamp) {
+ return asScratchTimestamp(elementNum).compareTo(timestamp);
}
/**
- * Return row as signed nanos (-999999999 to 999999999).
- * NOTE: This is not the same as the Timestamp class nanos (which is always positive).
+ * Compare Timestamp to row.
* We assume the entry has already been NULL checked and isRepeated adjusted.
+ * @param timestamp
* @param elementNum
- * @return
+ * @return -1, 0, 1 standard compareTo values.
*/
- public int getSignedNanos(int elementNum) {
- return PisaTimestamp.getSignedNanos(nanoOfDay[elementNum]);
+ public int compareTo(Timestamp timestamp, int elementNum) {
+ return timestamp.compareTo(asScratchTimestamp(elementNum));
}
/**
- * Get scratch timestamp with value of a row.
- * @param elementNum
+ * Compare a row to another TimestampColumnVector's row.
+ * @param elementNum1
+ * @param timestampColVector2
+ * @param elementNum2
* @return
*/
- public Timestamp asScratchTimestamp(int elementNum) {
- scratchPisaTimestamp.update(epochDay[elementNum], nanoOfDay[elementNum]);
- return scratchPisaTimestamp.asScratchTimestamp();
+ public int compareTo(int elementNum1, TimestampColumnVector timestampColVector2,
+ int elementNum2) {
+ return asScratchTimestamp(elementNum1).compareTo(
+ timestampColVector2.asScratchTimestamp(elementNum2));
}
/**
- * Get scratch Pisa timestamp for use by the caller.
+ * Compare another TimestampColumnVector's row to a row.
+ * @param timestampColVector1
+ * @param elementNum1
+ * @param elementNum2
* @return
*/
- public PisaTimestamp useScratchPisaTimestamp() {
- return scratchPisaTimestamp;
+ public int compareTo(TimestampColumnVector timestampColVector1, int elementNum1,
+ int elementNum2) {
+ return timestampColVector1.asScratchTimestamp(elementNum1).compareTo(
+ asScratchTimestamp(elementNum2));
}
@Override
@@ -290,8 +246,8 @@ public class TimestampColumnVector extends ColumnVector {
TimestampColumnVector timestampColVector = (TimestampColumnVector) inputVector;
- epochDay[outElementNum] = timestampColVector.epochDay[inputElementNum];
- nanoOfDay[outElementNum] = timestampColVector.nanoOfDay[inputElementNum];
+ time[outElementNum] = timestampColVector.time[inputElementNum];
+ nanos[outElementNum] = timestampColVector.nanos[inputElementNum];
}
// Simplify vector by brute-force flattening noNulls and isRepeating
@@ -301,17 +257,17 @@ public class TimestampColumnVector extends ColumnVector {
flattenPush();
if (isRepeating) {
isRepeating = false;
- long repeatEpochDay = epochDay[0];
- long repeatNanoOfDay = nanoOfDay[0];
+ long repeatFastTime = time[0];
+ int repeatNanos = nanos[0];
if (selectedInUse) {
for (int j = 0; j < size; j++) {
int i = sel[j];
- epochDay[i] = repeatEpochDay;
- nanoOfDay[i] = repeatNanoOfDay;
+ time[i] = repeatFastTime;
+ nanos[i] = repeatNanos;
}
} else {
- Arrays.fill(epochDay, 0, size, repeatEpochDay);
- Arrays.fill(nanoOfDay, 0, size, repeatNanoOfDay);
+ Arrays.fill(time, 0, size, repeatFastTime);
+ Arrays.fill(nanos, 0, size, repeatNanos);
}
flattenRepeatingNulls(selectedInUse, sel, size);
}
@@ -319,71 +275,23 @@ public class TimestampColumnVector extends ColumnVector {
}
/**
- * Set a row from a PisaTimestamp.
- * We assume the entry has already been isRepeated adjusted.
- * @param elementNum
- * @param pisaTimestamp
- */
- public void set(int elementNum, PisaTimestamp pisaTimestamp) {
- this.epochDay[elementNum] = pisaTimestamp.getEpochDay();
- this.nanoOfDay[elementNum] = pisaTimestamp.getNanoOfDay();
- }
-
- /**
* Set a row from a timestamp.
* We assume the entry has already been isRepeated adjusted.
* @param elementNum
* @param timestamp
*/
public void set(int elementNum, Timestamp timestamp) {
- scratchPisaTimestamp.updateFromTimestamp(timestamp);
- this.epochDay[elementNum] = scratchPisaTimestamp.getEpochDay();
- this.nanoOfDay[elementNum] = scratchPisaTimestamp.getNanoOfDay();
+ this.time[elementNum] = timestamp.getTime();
+ this.nanos[elementNum] = timestamp.getNanos();
}
/**
- * Set a row from a epoch seconds and signed nanos (-999999999 to 999999999).
+ * Set a row from the current value in the scratch timestamp.
* @param elementNum
- * @param epochSeconds
- * @param signedNanos
*/
- public void setEpochSecondsAndSignedNanos(int elementNum, long epochSeconds, int signedNanos) {
- scratchPisaTimestamp.updateFromEpochSecondsAndSignedNanos(epochSeconds, signedNanos);
- set(elementNum, scratchPisaTimestamp);
- }
-
- /**
- * Set a row from timestamp milliseconds.
- * We assume the entry has already been isRepeated adjusted.
- * @param elementNum
- * @param timestampMilliseconds
- */
- public void setTimestampMilliseconds(int elementNum, long timestampMilliseconds) {
- scratchPisaTimestamp.updateFromTimestampMilliseconds(timestampMilliseconds);
- set(elementNum, scratchPisaTimestamp.useScratchTimestamp());
- }
-
- /**
- * Set a row from timestamp seconds.
- * We assume the entry has already been isRepeated adjusted.
- * @param elementNum
- * @param timestamp
- */
- public void setTimestampSeconds(int elementNum, long timestampSeconds) {
- scratchPisaTimestamp.updateFromTimestampSeconds(timestampSeconds);
- set(elementNum, scratchPisaTimestamp);
- }
-
- /**
- * Set a row from a double timestamp seconds with fractional nanoseconds.
- * We assume the entry has already been isRepeated adjusted.
- * @param elementNum
- * @param timestamp
- */
- public void setTimestampSecondsWithFractionalNanoseconds(int elementNum,
- double secondsWithFractionalNanoseconds) {
- scratchPisaTimestamp.updateFromTimestampSecondsWithFractionalNanoseconds(secondsWithFractionalNanoseconds);
- set(elementNum, scratchPisaTimestamp);
+ public void setFromScratchTimestamp(int elementNum) {
+ this.time[elementNum] = scratchTimestamp.getTime();
+ this.nanos[elementNum] = scratchTimestamp.getNanos();
}
/**
@@ -392,8 +300,8 @@ public class TimestampColumnVector extends ColumnVector {
* @param elementNum
*/
public void setNullValue(int elementNum) {
- epochDay[elementNum] = 0;
- nanoOfDay[elementNum] = 1;
+ time[elementNum] = 0;
+ nanos[elementNum] = 1;
}
// Copy the current object contents into the output. Only copy selected entries,
@@ -407,8 +315,8 @@ public class TimestampColumnVector extends ColumnVector {
// Handle repeating case
if (isRepeating) {
- output.epochDay[0] = epochDay[0];
- output.nanoOfDay[0] = nanoOfDay[0];
+ output.time[0] = time[0];
+ output.nanos[0] = nanos[0];
output.isNull[0] = isNull[0];
output.isRepeating = true;
return;
@@ -420,13 +328,13 @@ public class TimestampColumnVector extends ColumnVector {
if (selectedInUse) {
for (int j = 0; j < size; j++) {
int i = sel[j];
- output.epochDay[i] = epochDay[i];
- output.nanoOfDay[i] = nanoOfDay[i];
+ output.time[i] = time[i];
+ output.nanos[i] = nanos[i];
}
}
else {
- System.arraycopy(epochDay, 0, output.epochDay, 0, size);
- System.arraycopy(nanoOfDay, 0, output.nanoOfDay, 0, size);
+ System.arraycopy(time, 0, output.time, 0, size);
+ System.arraycopy(nanos, 0, output.nanos, 0, size);
}
// Copy nulls over if needed
@@ -444,26 +352,14 @@ public class TimestampColumnVector extends ColumnVector {
}
/**
- * Fill all the vector entries with a PisaTimestamp.
- * @param pisaTimestamp
- */
- public void fill(PisaTimestamp pisaTimestamp) {
- noNulls = true;
- isRepeating = true;
- epochDay[0] = pisaTimestamp.getEpochDay();
- nanoOfDay[0] = pisaTimestamp.getNanoOfDay();
- }
-
- /**
* Fill all the vector entries with a timestamp.
* @param timestamp
*/
public void fill(Timestamp timestamp) {
noNulls = true;
isRepeating = true;
- scratchPisaTimestamp.updateFromTimestamp(timestamp);
- epochDay[0] = scratchPisaTimestamp.getEpochDay();
- nanoOfDay[0] = scratchPisaTimestamp.getNanoOfDay();
+ time[0] = timestamp.getTime();
+ nanos[0] = timestamp.getNanos();
}
/**
@@ -489,8 +385,9 @@ public class TimestampColumnVector extends ColumnVector {
row = 0;
}
if (noNulls || !isNull[row]) {
- scratchPisaTimestamp.update(epochDay[row], nanoOfDay[row]);
- buffer.append(scratchPisaTimestamp.toString());
+ scratchTimestamp.setTime(time[row]);
+ scratchTimestamp.setNanos(nanos[row]);
+ buffer.append(scratchTimestamp.toString());
} else {
buffer.append("null");
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/storage-api/src/java/org/apache/hive/common/util/IntervalDayTimeUtils.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hive/common/util/IntervalDayTimeUtils.java b/storage-api/src/java/org/apache/hive/common/util/IntervalDayTimeUtils.java
new file mode 100644
index 0000000..727c1e6
--- /dev/null
+++ b/storage-api/src/java/org/apache/hive/common/util/IntervalDayTimeUtils.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.common.util;
+
+import java.math.BigDecimal;
+import java.text.SimpleDateFormat;
+
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
+
+
+/**
+ * DateUtils. Thread-safe class
+ *
+ */
+public class IntervalDayTimeUtils {
+
+ private static final ThreadLocal<SimpleDateFormat> dateFormatLocal = new ThreadLocal<SimpleDateFormat>() {
+ @Override
+ protected SimpleDateFormat initialValue() {
+ return new SimpleDateFormat("yyyy-MM-dd");
+ }
+ };
+
+ public static SimpleDateFormat getDateFormat() {
+ return dateFormatLocal.get();
+ }
+
+ public static final int NANOS_PER_SEC = 1000000000;
+ public static final BigDecimal MAX_INT_BD = new BigDecimal(Integer.MAX_VALUE);
+ public static final BigDecimal NANOS_PER_SEC_BD = new BigDecimal(NANOS_PER_SEC);
+
+ public static int parseNumericValueWithRange(String fieldName,
+ String strVal, int minValue, int maxValue) throws IllegalArgumentException {
+ int result = 0;
+ if (strVal != null) {
+ result = Integer.parseInt(strVal);
+ if (result < minValue || result > maxValue) {
+ throw new IllegalArgumentException(String.format("%s value %d outside range [%d, %d]",
+ fieldName, result, minValue, maxValue));
+ }
+ }
+ return result;
+ }
+
+ public static long getIntervalDayTimeTotalNanos(HiveIntervalDayTime intervalDayTime) {
+ return intervalDayTime.getTotalSeconds() * NANOS_PER_SEC + intervalDayTime.getNanos();
+ }
+
+ public static void setIntervalDayTimeTotalNanos(HiveIntervalDayTime intervalDayTime,
+ long totalNanos) {
+ intervalDayTime.set(totalNanos / NANOS_PER_SEC, (int) (totalNanos % NANOS_PER_SEC));
+ }
+
+ public static long getIntervalDayTimeTotalSecondsFromTotalNanos(long totalNanos) {
+ return totalNanos / NANOS_PER_SEC;
+ }
+
+ public static int getIntervalDayTimeNanosFromTotalNanos(long totalNanos) {
+ return (int) (totalNanos % NANOS_PER_SEC);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/storage-api/src/test/org/apache/hadoop/hive/common/type/TestPisaTimestamp.java
----------------------------------------------------------------------
diff --git a/storage-api/src/test/org/apache/hadoop/hive/common/type/TestPisaTimestamp.java b/storage-api/src/test/org/apache/hadoop/hive/common/type/TestPisaTimestamp.java
deleted file mode 100644
index 8e7395c..0000000
--- a/storage-api/src/test/org/apache/hadoop/hive/common/type/TestPisaTimestamp.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.common.type;
-
-import org.junit.Test;
-
-import java.math.BigDecimal;
-import java.math.RoundingMode;
-import java.sql.Timestamp;
-import java.util.Random;
-import org.apache.hadoop.hive.common.type.RandomTypeUtil;
-
-import static org.junit.Assert.*;
-
-/**
- * Test for ListColumnVector
- */
-public class TestPisaTimestamp {
-
- private static int TEST_COUNT = 5000;
-
- @Test
- public void testPisaTimestampCreate() throws Exception {
-
- Random r = new Random(1234);
-
- for (int i = 0; i < TEST_COUNT; i++) {
- Timestamp randTimestamp = RandomTypeUtil.getRandTimestamp(r);
- PisaTimestamp pisaTimestamp = new PisaTimestamp(randTimestamp);
- Timestamp reconstructedTimestamp = new Timestamp(0);
- pisaTimestamp.timestampUpdate(reconstructedTimestamp);
- if (!randTimestamp.equals(reconstructedTimestamp)) {
- assertTrue(false);
- }
- }
- }
-
- static BigDecimal BIG_MAX_LONG = new BigDecimal(Long.MAX_VALUE);
- static BigDecimal BIG_MIN_LONG = new BigDecimal(Long.MIN_VALUE);
- static BigDecimal BIG_NANOSECONDS_PER_DAY = new BigDecimal(PisaTimestamp.NANOSECONDS_PER_DAY);
-
- static boolean beyondLongRange = false;
-
- private BigDecimal[] randomEpochDayAndNanoOfDay(Random r) {
- double randDouble = (r.nextDouble() - 0.5D) * 2.0D;
- randDouble *= PisaTimestamp.NANOSECONDS_PER_DAY;
- randDouble *= 365 * 10000;
- BigDecimal bigDecimal = new BigDecimal(randDouble);
- bigDecimal = bigDecimal.setScale(0, RoundingMode.HALF_UP);
-
- if (bigDecimal.compareTo(BIG_MAX_LONG) > 0 || bigDecimal.compareTo(BIG_MIN_LONG) < 0) {
- beyondLongRange = true;
- }
-
- BigDecimal[] divideAndRemainder = bigDecimal.divideAndRemainder(BIG_NANOSECONDS_PER_DAY);
-
- return new BigDecimal[] {divideAndRemainder[0], divideAndRemainder[1], bigDecimal};
- }
-
- private BigDecimal pisaTimestampToBig(PisaTimestamp pisaTimestamp) {
- BigDecimal bigNanoOfDay = new BigDecimal(pisaTimestamp.getNanoOfDay());
-
- BigDecimal bigEpochDay = new BigDecimal(pisaTimestamp.getEpochDay());
- BigDecimal result = bigEpochDay.multiply(BIG_NANOSECONDS_PER_DAY);
- result = result.add(bigNanoOfDay);
- return result;
- }
-
- @Test
- public void testPisaTimestampArithmetic() throws Exception {
-
- Random r = new Random(1234);
-
-
- for (int i = 0; i < TEST_COUNT; i++) {
- BigDecimal[] random1 = randomEpochDayAndNanoOfDay(r);
- long epochDay1 = random1[0].longValue();
- long nanoOfDay1 = random1[1].longValue();
- PisaTimestamp pisa1 = new PisaTimestamp(epochDay1, nanoOfDay1);
- BigDecimal big1 = random1[2];
-
- BigDecimal[] random2 = randomEpochDayAndNanoOfDay(r);
- long epochDay2 = random2[0].longValue();
- long nanoOfDay2 = random2[1].longValue();
- PisaTimestamp pisa2 = new PisaTimestamp(epochDay2, nanoOfDay2);
- BigDecimal big2 = random2[2];
-
- BigDecimal expectedBig;
- PisaTimestamp pisaResult = new PisaTimestamp();
- if (i % 2 == 0) {
- expectedBig = big1.add(big2);
- PisaTimestamp.add(pisa1, pisa2, pisaResult);
- } else {
- expectedBig = big1.add(big2.negate());
- PisaTimestamp.subtract(pisa1, pisa2, pisaResult);
- }
- BigDecimal resultBig = pisaTimestampToBig(pisaResult);
- assertEquals(expectedBig, resultBig);
-
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestTimestampColumnVector.java
----------------------------------------------------------------------
diff --git a/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestTimestampColumnVector.java b/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestTimestampColumnVector.java
new file mode 100644
index 0000000..6e5d5c8
--- /dev/null
+++ b/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestTimestampColumnVector.java
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector;
+
+import org.junit.Test;
+
+import java.io.PrintWriter;
+import java.math.BigDecimal;
+import java.math.RoundingMode;
+import java.sql.Timestamp;
+import java.util.Date;
+import java.util.Random;
+
+import org.apache.hadoop.hive.common.type.RandomTypeUtil;
+
+import static org.junit.Assert.*;
+
+/**
+ * Test for ListColumnVector
+ */
+public class TestTimestampColumnVector {
+
+ private static int TEST_COUNT = 5000;
+
+ private static int fake = 0;
+
+ @Test
+ public void testSaveAndRetrieve() throws Exception {
+
+ Random r = new Random(1234);
+ TimestampColumnVector timestampColVector = new TimestampColumnVector();
+ Timestamp[] randTimestamps = new Timestamp[VectorizedRowBatch.DEFAULT_SIZE];
+
+ for (int i = 0; i < VectorizedRowBatch.DEFAULT_SIZE; i++) {
+ Timestamp randTimestamp = RandomTypeUtil.getRandTimestamp(r);
+ randTimestamps[i] = randTimestamp;
+ timestampColVector.set(i, randTimestamp);
+ }
+ for (int i = 0; i < VectorizedRowBatch.DEFAULT_SIZE; i++) {
+ Timestamp retrievedTimestamp = timestampColVector.asScratchTimestamp(i);
+ Timestamp randTimestamp = randTimestamps[i];
+ if (!retrievedTimestamp.equals(randTimestamp)) {
+ assertTrue(false);
+ }
+ }
+ }
+
+ @Test
+ public void testTimestampCompare() throws Exception {
+ Random r = new Random(1234);
+ TimestampColumnVector timestampColVector = new TimestampColumnVector();
+ Timestamp[] randTimestamps = new Timestamp[VectorizedRowBatch.DEFAULT_SIZE];
+ Timestamp[] candTimestamps = new Timestamp[VectorizedRowBatch.DEFAULT_SIZE];
+ int[] compareToLeftRights = new int[VectorizedRowBatch.DEFAULT_SIZE];
+ int[] compareToRightLefts = new int[VectorizedRowBatch.DEFAULT_SIZE];
+
+ for (int i = 0; i < VectorizedRowBatch.DEFAULT_SIZE; i++) {
+ Timestamp randTimestamp = RandomTypeUtil.getRandTimestamp(r);
+ randTimestamps[i] = randTimestamp;
+ timestampColVector.set(i, randTimestamp);
+ Timestamp candTimestamp = RandomTypeUtil.getRandTimestamp(r);
+ candTimestamps[i] = candTimestamp;
+ compareToLeftRights[i] = candTimestamp.compareTo(randTimestamp);
+ compareToRightLefts[i] = randTimestamp.compareTo(candTimestamp);
+ }
+
+ for (int i = 0; i < VectorizedRowBatch.DEFAULT_SIZE; i++) {
+ Timestamp retrievedTimestamp = timestampColVector.asScratchTimestamp(i);
+ Timestamp randTimestamp = randTimestamps[i];
+ if (!retrievedTimestamp.equals(randTimestamp)) {
+ assertTrue(false);
+ }
+ Timestamp candTimestamp = candTimestamps[i];
+ int compareToLeftRight = timestampColVector.compareTo(candTimestamp, i);
+ if (compareToLeftRight != compareToLeftRights[i]) {
+ assertTrue(false);
+ }
+ int compareToRightLeft = timestampColVector.compareTo(i, candTimestamp);
+ if (compareToRightLeft != compareToRightLefts[i]) {
+ assertTrue(false);
+ }
+ }
+ }
+
+ /*
+ @Test
+ public void testGenerate() throws Exception {
+ PrintWriter writer = new PrintWriter("/Users/you/timestamps.txt");
+ Random r = new Random(18485);
+ for (int i = 0; i < 25; i++) {
+ Timestamp randTimestamp = RandomTypeUtil.getRandTimestamp(r);
+ writer.println(randTimestamp.toString());
+ }
+ for (int i = 0; i < 25; i++) {
+ Timestamp randTimestamp = RandomTypeUtil.getRandTimestamp(r, 1965, 2025);
+ writer.println(randTimestamp.toString());
+ }
+ writer.close();
+ }
+ */
+}
[39/50] [abbrv] hive git commit: HIVE-11424 : Rule to transform OR
clauses into IN clauses in CBO (Jesus Camacho Rodriguez via Ashutosh Chauhan)
Posted by jd...@apache.org.
HIVE-11424 : Rule to transform OR clauses into IN clauses in CBO (Jesus Camacho Rodriguez via Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8c8ff3f1
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8c8ff3f1
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8c8ff3f1
Branch: refs/heads/llap
Commit: 8c8ff3f144921e9b985abe51eb82ebad94195b4a
Parents: 09b00fc
Author: Jesus Camacho Rodriguez <jc...@apache.org>
Authored: Tue Mar 22 23:41:00 2016 -0800
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Tue Mar 29 11:18:58 2016 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/exec/FunctionRegistry.java | 7 +
.../hadoop/hive/ql/optimizer/Optimizer.java | 4 +-
.../rules/HivePointLookupOptimizerRule.java | 381 +++++++++++++++++++
.../ql/optimizer/pcr/PcrExprProcFactory.java | 103 ++---
.../hadoop/hive/ql/parse/CalcitePlanner.java | 40 +-
.../clientpositive/auto_join19_inclause.q | 18 +
.../queries/clientpositive/filter_in_or_dup.q | 19 +
.../clientpositive/auto_join19_inclause.q.out | 130 +++++++
.../clientpositive/constprog_semijoin.q.out | 4 +-
.../dynpart_sort_optimization_acid.q.out | 4 +-
.../clientpositive/filter_in_or_dup.q.out | 96 +++++
.../results/clientpositive/perf/query13.q.out | 14 +-
.../results/clientpositive/perf/query27.q.out | 2 +-
.../results/clientpositive/perf/query34.q.out | 2 +-
.../results/clientpositive/perf/query48.q.out | 14 +-
.../results/clientpositive/perf/query68.q.out | 2 +-
.../results/clientpositive/perf/query73.q.out | 2 +-
.../results/clientpositive/perf/query79.q.out | 2 +-
.../results/clientpositive/perf/query82.q.out | 2 +-
.../results/clientpositive/perf/query85.q.out | 26 +-
.../results/clientpositive/pointlookup2.q.out | 38 +-
.../results/clientpositive/pointlookup3.q.out | 50 ++-
.../results/clientpositive/pointlookup4.q.out | 2 +-
.../spark/constprog_semijoin.q.out | 4 +-
.../clientpositive/tez/bucketpruning1.q.out | 8 +-
.../clientpositive/tez/constprog_semijoin.q.out | 4 +-
.../tez/vector_mr_diff_schema_alias.q.out | 2 +-
.../vector_mr_diff_schema_alias.q.out | 2 +-
28 files changed, 824 insertions(+), 158 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/8c8ff3f1/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
index b516925..56b96b4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
@@ -1398,6 +1398,13 @@ public final class FunctionRegistry {
}
/**
+ * Returns whether the exprNodeDesc is a node of "in".
+ */
+ public static boolean isIn(ExprNodeDesc desc) {
+ return GenericUDFIn.class == getGenericUDFClassFromExprDesc(desc);
+ }
+
+ /**
* Returns whether the exprNodeDesc is a node of "not".
*/
public static boolean isOpNot(ExprNodeDesc desc) {
http://git-wip-us.apache.org/repos/asf/hive/blob/8c8ff3f1/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
index f56cd96..55c71dd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
@@ -23,7 +23,6 @@ import java.util.List;
import java.util.Set;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.optimizer.ConstantPropagateProcCtx.ConstantPropagateOption;
import org.apache.hadoop.hive.ql.optimizer.calcite.translator.HiveOpConverterPostProc;
import org.apache.hadoop.hive.ql.optimizer.correlation.CorrelationOptimizer;
import org.apache.hadoop.hive.ql.optimizer.correlation.ReduceSinkDeDuplication;
@@ -83,7 +82,8 @@ public class Optimizer {
}
// Try to transform OR predicates in Filter into simpler IN clauses first
- if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEPOINTLOOKUPOPTIMIZER)) {
+ if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEPOINTLOOKUPOPTIMIZER) &&
+ !pctx.getContext().isCboSucceeded()) {
final int min = HiveConf.getIntVar(hiveConf,
HiveConf.ConfVars.HIVEPOINTLOOKUPOPTIMIZERMIN);
transformations.add(new PointLookupOptimizer(min));
http://git-wip-us.apache.org/repos/asf/hive/blob/8c8ff3f1/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePointLookupOptimizerRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePointLookupOptimizerRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePointLookupOptimizerRule.java
new file mode 100644
index 0000000..9609a1e
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePointLookupOptimizerRule.java
@@ -0,0 +1,381 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.optimizer.calcite.rules;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.plan.RelOptUtil;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.Filter;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexShuttle;
+import org.apache.calcite.rex.RexUtil;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveIn;
+import org.apache.hadoop.hive.ql.optimizer.calcite.translator.TypeConverter;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.LinkedHashMultimap;
+import com.google.common.collect.ListMultimap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Sets;
+
+/**
+ * This optimization will take a Filter expression, and if its predicate contains
+ * an OR operator whose children are constant equality expressions, it will try
+ * to generate an IN clause (which is more efficient). If the OR operator contains
+ * AND operator children, the optimization might generate an IN clause that uses
+ * structs.
+ */
+public class HivePointLookupOptimizerRule extends RelOptRule {
+
+ protected static final Log LOG = LogFactory.getLog(HivePointLookupOptimizerRule.class);
+
+
+ // Minimum number of OR clauses needed to transform into IN clauses
+ private final int min;
+
+ public HivePointLookupOptimizerRule(int min) {
+ super(operand(Filter.class, any()));
+ this.min = min;
+ }
+
+ public void onMatch(RelOptRuleCall call) {
+ final Filter filter = call.rel(0);
+
+ final RexBuilder rexBuilder = filter.getCluster().getRexBuilder();
+
+ final RexNode condition = RexUtil.pullFactors(rexBuilder, filter.getCondition());
+
+ // 1. We try to transform possible candidates
+ RexTransformIntoInClause transformIntoInClause = new RexTransformIntoInClause(rexBuilder, filter, min);
+ RexNode newCondition = transformIntoInClause.apply(condition);
+
+ // 2. We merge IN expressions
+ RexMergeInClause mergeInClause = new RexMergeInClause(rexBuilder);
+ newCondition = mergeInClause.apply(newCondition);
+
+ // 3. If we could not transform anything, we bail out
+ if (newCondition.toString().equals(condition.toString())) {
+ return;
+ }
+
+ // 4. We create the filter with the new condition
+ RelNode newFilter = filter.copy(filter.getTraitSet(), filter.getInput(), newCondition);
+
+ call.transformTo(newFilter);
+ }
+
+
+ /**
+ * Transforms OR clauses into IN clauses, when possible.
+ */
+ protected static class RexTransformIntoInClause extends RexShuttle {
+ private final RexBuilder rexBuilder;
+ private final Filter filterOp;
+ private final int min;
+
+ RexTransformIntoInClause(RexBuilder rexBuilder, Filter filterOp, int min) {
+ this.filterOp = filterOp;
+ this.rexBuilder = rexBuilder;
+ this.min = min;
+ }
+
+ @Override public RexNode visitCall(RexCall call) {
+ RexNode node;
+ switch (call.getKind()) {
+ case AND:
+ ImmutableList<RexNode> operands = RexUtil.flattenAnd(((RexCall) call).getOperands());
+ List<RexNode> newOperands = new ArrayList<RexNode>();
+ for (RexNode operand: operands) {
+ RexNode newOperand;
+ if (operand.getKind() == SqlKind.OR) {
+ try {
+ newOperand = transformIntoInClauseCondition(rexBuilder,
+ filterOp.getRowType(), operand, min);
+ if (newOperand == null) {
+ return call;
+ }
+ } catch (SemanticException e) {
+ LOG.error("Exception in HivePointLookupOptimizerRule", e);
+ return call;
+ }
+ } else {
+ newOperand = operand;
+ }
+ newOperands.add(newOperand);
+ }
+ node = RexUtil.composeConjunction(rexBuilder, newOperands, false);
+ break;
+ case OR:
+ try {
+ node = transformIntoInClauseCondition(rexBuilder,
+ filterOp.getRowType(), call, min);
+ if (node == null) {
+ return call;
+ }
+ } catch (SemanticException e) {
+ LOG.error("Exception in HivePointLookupOptimizerRule", e);
+ return call;
+ }
+ break;
+ default:
+ return super.visitCall(call);
+ }
+ return node;
+ }
+
+ private static RexNode transformIntoInClauseCondition(RexBuilder rexBuilder, RelDataType inputSchema,
+ RexNode condition, int min) throws SemanticException {
+ assert condition.getKind() == SqlKind.OR;
+
+ // 1. We extract the information necessary to create the predicate for the new
+ // filter
+ ListMultimap<RexInputRef,RexLiteral> columnConstantsMap = ArrayListMultimap.create();
+ ImmutableList<RexNode> operands = RexUtil.flattenOr(((RexCall) condition).getOperands());
+ if (operands.size() < min) {
+ // We bail out
+ return null;
+ }
+ for (int i = 0; i < operands.size(); i++) {
+ RexNode operand = operands.get(i);
+
+ final RexNode operandCNF = RexUtil.toCnf(rexBuilder, operand);
+ final List<RexNode> conjunctions = RelOptUtil.conjunctions(operandCNF);
+
+ for (RexNode conjunction: conjunctions) {
+ // 1.1. If it is not a RexCall, we bail out
+ if (!(conjunction instanceof RexCall)) {
+ return null;
+ }
+ // 1.2. We extract the information that we need
+ RexCall conjCall = (RexCall) conjunction;
+ if(conjCall.getOperator().getKind() == SqlKind.EQUALS) {
+ if (conjCall.operands.get(0) instanceof RexInputRef &&
+ conjCall.operands.get(1) instanceof RexLiteral) {
+ RexInputRef ref = (RexInputRef) conjCall.operands.get(0);
+ RexLiteral literal = (RexLiteral) conjCall.operands.get(1);
+ columnConstantsMap.put(ref, literal);
+ if (columnConstantsMap.get(ref).size() != i+1) {
+ // If we have not added to this column before, we bail out
+ return null;
+ }
+ } else if (conjCall.operands.get(1) instanceof RexInputRef &&
+ conjCall.operands.get(0) instanceof RexLiteral) {
+ RexInputRef ref = (RexInputRef) conjCall.operands.get(1);
+ RexLiteral literal = (RexLiteral) conjCall.operands.get(0);
+ columnConstantsMap.put(ref, literal);
+ if (columnConstantsMap.get(ref).size() != i+1) {
+ // If we have not added to this column before, we bail out
+ return null;
+ }
+ } else {
+ // Bail out
+ return null;
+ }
+ } else {
+ return null;
+ }
+ }
+ }
+
+ // 3. We build the new predicate and return it
+ List<RexNode> newOperands = new ArrayList<RexNode>(operands.size());
+ // 3.1 Create structs
+ List<RexInputRef> columns = new ArrayList<RexInputRef>();
+ List<String> names = new ArrayList<String>();
+ ImmutableList.Builder<RelDataType> paramsTypes = ImmutableList.builder();
+ List<TypeInfo> structReturnType = new ArrayList<TypeInfo>();
+ ImmutableList.Builder<RelDataType> newOperandsTypes = ImmutableList.builder();
+ for (int i = 0; i < operands.size(); i++) {
+ List<RexLiteral> constantFields = new ArrayList<RexLiteral>(operands.size());
+
+ for (RexInputRef ref : columnConstantsMap.keySet()) {
+ // If any of the elements was not referenced by every operand, we bail out
+ if (columnConstantsMap.get(ref).size() <= i) {
+ return null;
+ }
+ RexLiteral columnConstant = columnConstantsMap.get(ref).get(i);
+ if (i == 0) {
+ columns.add(ref);
+ names.add(inputSchema.getFieldNames().get(ref.getIndex()));
+ paramsTypes.add(ref.getType());
+ structReturnType.add(TypeConverter.convert(ref.getType()));
+ }
+ constantFields.add(columnConstant);
+ }
+
+ if (i == 0) {
+ RexNode columnsRefs;
+ if (columns.size() == 1) {
+ columnsRefs = columns.get(0);
+ } else {
+ // Create STRUCT clause
+ columnsRefs = rexBuilder.makeCall(SqlStdOperatorTable.ROW, columns);
+ }
+ newOperands.add(columnsRefs);
+ newOperandsTypes.add(columnsRefs.getType());
+ }
+ RexNode values;
+ if (constantFields.size() == 1) {
+ values = constantFields.get(0);
+ } else {
+ // Create STRUCT clause
+ values = rexBuilder.makeCall(SqlStdOperatorTable.ROW, constantFields);
+ }
+ newOperands.add(values);
+ newOperandsTypes.add(values.getType());
+ }
+
+ // 4. Create and return IN clause
+ return rexBuilder.makeCall(HiveIn.INSTANCE, newOperands);
+ }
+
+ }
+
+ /**
+ * Merge IN clauses, when possible.
+ */
+ protected static class RexMergeInClause extends RexShuttle {
+ private final RexBuilder rexBuilder;
+
+ RexMergeInClause(RexBuilder rexBuilder) {
+ this.rexBuilder = rexBuilder;
+ }
+
+ @Override public RexNode visitCall(RexCall call) {
+ RexNode node;
+ final List<RexNode> operands;
+ final List<RexNode> newOperands;
+ Map<String,RexNode> stringToExpr = Maps.newHashMap();
+ Multimap<String,String> inLHSExprToRHSExprs = LinkedHashMultimap.create();
+ switch (call.getKind()) {
+ case AND:
+ // IN clauses need to be combined by keeping only common elements
+ operands = Lists.newArrayList(RexUtil.flattenAnd(((RexCall) call).getOperands()));
+ for (int i = 0; i < operands.size(); i++) {
+ RexNode operand = operands.get(i);
+ if (operand.getKind() == SqlKind.IN) {
+ RexCall inCall = (RexCall) operand;
+ if (!HiveCalciteUtil.isDeterministic(inCall.getOperands().get(0))) {
+ continue;
+ }
+ String ref = inCall.getOperands().get(0).toString();
+ stringToExpr.put(ref, inCall.getOperands().get(0));
+ if (inLHSExprToRHSExprs.containsKey(ref)) {
+ Set<String> expressions = Sets.newHashSet();
+ for (int j = 1; j < inCall.getOperands().size(); j++) {
+ String expr = inCall.getOperands().get(j).toString();
+ expressions.add(expr);
+ stringToExpr.put(expr, inCall.getOperands().get(j));
+ }
+ inLHSExprToRHSExprs.get(ref).retainAll(expressions);
+ } else {
+ for (int j = 1; j < inCall.getOperands().size(); j++) {
+ String expr = inCall.getOperands().get(j).toString();
+ inLHSExprToRHSExprs.put(ref, expr);
+ stringToExpr.put(expr, inCall.getOperands().get(j));
+ }
+ }
+ operands.remove(i);
+ --i;
+ }
+ }
+ // Create IN clauses
+ newOperands = createInClauses(rexBuilder, stringToExpr, inLHSExprToRHSExprs);
+ newOperands.addAll(operands);
+ // Return node
+ node = RexUtil.composeConjunction(rexBuilder, newOperands, false);
+ break;
+ case OR:
+ // IN clauses need to be combined by keeping all elements
+ operands = Lists.newArrayList(RexUtil.flattenOr(((RexCall) call).getOperands()));
+ for (int i = 0; i < operands.size(); i++) {
+ RexNode operand = operands.get(i);
+ if (operand.getKind() == SqlKind.IN) {
+ RexCall inCall = (RexCall) operand;
+ if (!HiveCalciteUtil.isDeterministic(inCall.getOperands().get(0))) {
+ continue;
+ }
+ String ref = inCall.getOperands().get(0).toString();
+ stringToExpr.put(ref, inCall.getOperands().get(0));
+ for (int j = 1; j < inCall.getOperands().size(); j++) {
+ String expr = inCall.getOperands().get(j).toString();
+ inLHSExprToRHSExprs.put(ref, expr);
+ stringToExpr.put(expr, inCall.getOperands().get(j));
+ }
+ operands.remove(i);
+ --i;
+ }
+ }
+ // Create IN clauses
+ newOperands = createInClauses(rexBuilder, stringToExpr, inLHSExprToRHSExprs);
+ newOperands.addAll(operands);
+ // Return node
+ node = RexUtil.composeDisjunction(rexBuilder, newOperands, false);
+ break;
+ default:
+ return super.visitCall(call);
+ }
+ return node;
+ }
+
+ private static List<RexNode> createInClauses(RexBuilder rexBuilder, Map<String, RexNode> stringToExpr,
+ Multimap<String, String> inLHSExprToRHSExprs) {
+ List<RexNode> newExpressions = Lists.newArrayList();
+ for (Entry<String,Collection<String>> entry : inLHSExprToRHSExprs.asMap().entrySet()) {
+ String ref = entry.getKey();
+ Collection<String> exprs = entry.getValue();
+ if (exprs.isEmpty()) {
+ newExpressions.add(rexBuilder.makeLiteral(false));
+ } else {
+ List<RexNode> newOperands = new ArrayList<RexNode>(exprs.size() + 1);
+ newOperands.add(stringToExpr.get(ref));
+ for (String expr : exprs) {
+ newOperands.add(stringToExpr.get(expr));
+ }
+ newExpressions.add(rexBuilder.makeCall(HiveIn.INSTANCE, newOperands));
+ }
+ }
+ return newExpressions;
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/8c8ff3f1/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PcrExprProcFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PcrExprProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PcrExprProcFactory.java
index 9cc9ea9..9911179 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PcrExprProcFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PcrExprProcFactory.java
@@ -25,8 +25,6 @@ import java.util.List;
import java.util.Map;
import java.util.Stack;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker;
import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher;
@@ -49,13 +47,12 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDynamicListDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeFieldDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFIn;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFStruct;
import org.apache.hadoop.hive.serde2.SerDeException;
-import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Expression processor factory for partition condition removing. Each processor tries to
@@ -368,50 +365,66 @@ public final class PcrExprProcFactory {
return getResultWrapFromResults(results, fd, newNodeOutputs);
}
return new NodeInfoWrapper(WalkState.UNKNOWN, null, getOutExpr(fd, newNodeOutputs));
- } else if (fd.getGenericUDF() instanceof GenericUDFIn) {
- List<ExprNodeDesc> children = fd.getChildren();
- boolean removePredElem = false;
- ExprNodeDesc lhs = children.get(0);
-
- if (lhs instanceof ExprNodeGenericFuncDesc) {
- // Make sure that the generic udf is deterministic
- if (FunctionRegistry.isDeterministic(((ExprNodeGenericFuncDesc) lhs)
- .getGenericUDF())) {
- boolean hasOnlyPartCols = true;
- boolean hasDynamicListDesc = false;
-
- for (ExprNodeDesc ed : ((ExprNodeGenericFuncDesc) lhs).getChildren()) {
- // Check if the current field expression contains only
- // partition column or a virtual column or constants.
- // If yes, this filter predicate is a candidate for this optimization.
- if (!(ed instanceof ExprNodeColumnDesc &&
- ((ExprNodeColumnDesc)ed).getIsPartitionColOrVirtualCol())) {
- hasOnlyPartCols = false;
- break;
- }
- }
+ } else if (FunctionRegistry.isIn(fd)) {
+ List<ExprNodeDesc> children = fd.getChildren();
+ boolean removePredElem = false;
+ ExprNodeDesc lhs = children.get(0);
+
+ if (lhs instanceof ExprNodeColumnDesc) {
+ // It is an IN clause on a column
+ if (((ExprNodeColumnDesc)lhs).getIsPartitionColOrVirtualCol()) {
+ // It is a partition column, we can proceed
+ removePredElem = true;
+ }
+ if (removePredElem) {
+ // We should not remove the dynamic partition pruner generated synthetic predicates.
+ for (int i = 1; i < children.size(); i++) {
+ if (children.get(i) instanceof ExprNodeDynamicListDesc) {
+ removePredElem = false;
+ break;
+ }
+ }
+ }
+ } else if (lhs instanceof ExprNodeGenericFuncDesc) {
+ // It is an IN clause on a struct
+ // Make sure that the generic udf is deterministic
+ if (FunctionRegistry.isDeterministic(((ExprNodeGenericFuncDesc) lhs)
+ .getGenericUDF())) {
+ boolean hasOnlyPartCols = true;
+ boolean hasDynamicListDesc = false;
+
+ for (ExprNodeDesc ed : ((ExprNodeGenericFuncDesc) lhs).getChildren()) {
+ // Check if the current field expression contains only
+ // partition column or a virtual column or constants.
+ // If yes, this filter predicate is a candidate for this optimization.
+ if (!(ed instanceof ExprNodeColumnDesc &&
+ ((ExprNodeColumnDesc)ed).getIsPartitionColOrVirtualCol())) {
+ hasOnlyPartCols = false;
+ break;
+ }
+ }
- // If we have non-partition columns, we cannot remove the predicate.
- if (hasOnlyPartCols) {
- // We should not remove the dynamic partition pruner generated synthetic predicates.
- for (int i = 1; i < children.size(); i++) {
- if (children.get(i) instanceof ExprNodeDynamicListDesc) {
- hasDynamicListDesc = true;
- break;
- }
- }
+ // If we have non-partition columns, we cannot remove the predicate.
+ if (hasOnlyPartCols) {
+ // We should not remove the dynamic partition pruner generated synthetic predicates.
+ for (int i = 1; i < children.size(); i++) {
+ if (children.get(i) instanceof ExprNodeDynamicListDesc) {
+ hasDynamicListDesc = true;
+ break;
}
-
- removePredElem = hasOnlyPartCols && !hasDynamicListDesc;
}
+ }
+
+ removePredElem = hasOnlyPartCols && !hasDynamicListDesc;
}
+ }
- // If removePredElem is set to true, return true as this is a potential candidate
- // for partition condition remover. Else, set the WalkState for this node to unknown.
- return removePredElem ?
- new NodeInfoWrapper(WalkState.TRUE, null,
- new ExprNodeConstantDesc(fd.getTypeInfo(), Boolean.TRUE)) :
- new NodeInfoWrapper(WalkState.UNKNOWN, null, getOutExpr(fd, nodeOutputs)) ;
+ // If removePredElem is set to true, return true as this is a potential candidate
+ // for partition condition remover. Else, set the WalkState for this node to unknown.
+ return removePredElem ?
+ new NodeInfoWrapper(WalkState.TRUE, null,
+ new ExprNodeConstantDesc(fd.getTypeInfo(), Boolean.TRUE)) :
+ new NodeInfoWrapper(WalkState.UNKNOWN, null, getOutExpr(fd, nodeOutputs)) ;
} else if (!FunctionRegistry.isDeterministic(fd.getGenericUDF())) {
// If it's a non-deterministic UDF, set unknown to true
return new NodeInfoWrapper(WalkState.UNKNOWN, null,
http://git-wip-us.apache.org/repos/asf/hive/blob/8c8ff3f1/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index fd2246b..b59347d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -153,6 +153,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveJoinProjectTranspos
import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveJoinPushTransitivePredicatesRule;
import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveJoinToMultiJoinRule;
import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HivePartitionPruneRule;
+import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HivePointLookupOptimizerRule;
import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HivePreFilteringRule;
import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveProjectMergeRule;
import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveProjectSortTransposeRule;
@@ -1138,23 +1139,32 @@ public class CalcitePlanner extends SemanticAnalyzer {
// 3. Run exhaustive PPD, add not null filters, transitive inference,
// constant propagation, constant folding
+ List<RelOptRule> rules = Lists.newArrayList();
+ if (conf.getBoolVar(HiveConf.ConfVars.HIVEOPTPPD_WINDOWING)) {
+ rules.add(HiveFilterProjectTransposeRule.INSTANCE_DETERMINISTIC_WINDOWING);
+ } else {
+ rules.add(HiveFilterProjectTransposeRule.INSTANCE_DETERMINISTIC);
+ }
+ rules.add(HiveFilterSetOpTransposeRule.INSTANCE);
+ rules.add(HiveFilterSortTransposeRule.INSTANCE);
+ rules.add(HiveFilterJoinRule.JOIN);
+ rules.add(HiveFilterJoinRule.FILTER_ON_JOIN);
+ rules.add(new HiveFilterAggregateTransposeRule(Filter.class, HiveRelFactories.HIVE_FILTER_FACTORY, Aggregate.class));
+ rules.add(new FilterMergeRule(HiveRelFactories.HIVE_FILTER_FACTORY));
+ rules.add(HiveReduceExpressionsRule.PROJECT_INSTANCE);
+ rules.add(HiveReduceExpressionsRule.FILTER_INSTANCE);
+ rules.add(HiveReduceExpressionsRule.JOIN_INSTANCE);
+ if (conf.getBoolVar(HiveConf.ConfVars.HIVEPOINTLOOKUPOPTIMIZER)) {
+ final int min = conf.getIntVar(HiveConf.ConfVars.HIVEPOINTLOOKUPOPTIMIZERMIN);
+ rules.add(new HivePointLookupOptimizerRule(min));
+ }
+ rules.add(HiveJoinAddNotNullRule.INSTANCE_JOIN);
+ rules.add(HiveJoinAddNotNullRule.INSTANCE_SEMIJOIN);
+ rules.add(HiveJoinPushTransitivePredicatesRule.INSTANCE_JOIN);
+ rules.add(HiveJoinPushTransitivePredicatesRule.INSTANCE_SEMIJOIN);
perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER);
basePlan = hepPlan(basePlan, true, mdProvider, executorProvider, HepMatchOrder.BOTTOM_UP,
- conf.getBoolVar(HiveConf.ConfVars.HIVEOPTPPD_WINDOWING) ? HiveFilterProjectTransposeRule.INSTANCE_DETERMINISTIC_WINDOWING
- : HiveFilterProjectTransposeRule.INSTANCE_DETERMINISTIC,
- HiveFilterSetOpTransposeRule.INSTANCE,
- HiveFilterSortTransposeRule.INSTANCE,
- HiveFilterJoinRule.JOIN,
- HiveFilterJoinRule.FILTER_ON_JOIN,
- new HiveFilterAggregateTransposeRule(Filter.class, HiveRelFactories.HIVE_FILTER_FACTORY, Aggregate.class),
- new FilterMergeRule(HiveRelFactories.HIVE_FILTER_FACTORY),
- HiveReduceExpressionsRule.PROJECT_INSTANCE,
- HiveReduceExpressionsRule.FILTER_INSTANCE,
- HiveReduceExpressionsRule.JOIN_INSTANCE,
- HiveJoinAddNotNullRule.INSTANCE_JOIN,
- HiveJoinAddNotNullRule.INSTANCE_SEMIJOIN,
- HiveJoinPushTransitivePredicatesRule.INSTANCE_JOIN,
- HiveJoinPushTransitivePredicatesRule.INSTANCE_SEMIJOIN);
+ rules.toArray(new RelOptRule[rules.size()]));
perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER,
"Calcite: Prejoin ordering transformation, PPD, not null predicates, transitive inference, constant folding");
http://git-wip-us.apache.org/repos/asf/hive/blob/8c8ff3f1/ql/src/test/queries/clientpositive/auto_join19_inclause.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/auto_join19_inclause.q b/ql/src/test/queries/clientpositive/auto_join19_inclause.q
new file mode 100644
index 0000000..7773289
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/auto_join19_inclause.q
@@ -0,0 +1,18 @@
+set hive.mapred.mode=nonstrict;
+set hive.auto.convert.join = true;
+set hive.optimize.point.lookup.min=2;
+
+CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE;
+
+explain
+FROM srcpart src1 JOIN src src2 ON (src1.key = src2.key)
+INSERT OVERWRITE TABLE dest1 SELECT src1.key, src2.value
+where (src1.ds = '2008-04-08' or src1.ds = '2008-04-09' )and (src1.hr = '12' or src1.hr = '11');
+
+
+FROM srcpart src1 JOIN src src2 ON (src1.key = src2.key)
+INSERT OVERWRITE TABLE dest1 SELECT src1.key, src2.value
+where (src1.ds = '2008-04-08' or src1.ds = '2008-04-09' )and (src1.hr = '12' or src1.hr = '11');
+
+
+SELECT sum(hash(dest1.key,dest1.value)) FROM dest1;
http://git-wip-us.apache.org/repos/asf/hive/blob/8c8ff3f1/ql/src/test/queries/clientpositive/filter_in_or_dup.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/filter_in_or_dup.q b/ql/src/test/queries/clientpositive/filter_in_or_dup.q
new file mode 100644
index 0000000..34a5139
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/filter_in_or_dup.q
@@ -0,0 +1,19 @@
+set hive.optimize.point.lookup.min=2;
+
+EXPLAIN
+SELECT f.key
+FROM cbo_t1 f
+WHERE (f.key = '1' OR f.key='2')
+AND f.key IN ('1', '2');
+
+EXPLAIN
+SELECT f.key
+FROM cbo_t1 f
+WHERE (f.key = '1' OR f.key = '2')
+AND f.key IN ('1', '2', '3');
+
+EXPLAIN
+SELECT f.key
+FROM cbo_t1 f
+WHERE (f.key = '1' OR f.key='2' OR f.key='3')
+AND f.key IN ('1', '2');
http://git-wip-us.apache.org/repos/asf/hive/blob/8c8ff3f1/ql/src/test/results/clientpositive/auto_join19_inclause.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/auto_join19_inclause.q.out b/ql/src/test/results/clientpositive/auto_join19_inclause.q.out
new file mode 100644
index 0000000..3f70055
--- /dev/null
+++ b/ql/src/test/results/clientpositive/auto_join19_inclause.q.out
@@ -0,0 +1,130 @@
+PREHOOK: query: CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@dest1
+POSTHOOK: query: CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@dest1
+PREHOOK: query: explain
+FROM srcpart src1 JOIN src src2 ON (src1.key = src2.key)
+INSERT OVERWRITE TABLE dest1 SELECT src1.key, src2.value
+where (src1.ds = '2008-04-08' or src1.ds = '2008-04-09' )and (src1.hr = '12' or src1.hr = '11')
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+FROM srcpart src1 JOIN src src2 ON (src1.key = src2.key)
+INSERT OVERWRITE TABLE dest1 SELECT src1.key, src2.value
+where (src1.ds = '2008-04-08' or src1.ds = '2008-04-09' )and (src1.hr = '12' or src1.hr = '11')
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-5 is a root stage
+ Stage-4 depends on stages: Stage-5
+ Stage-0 depends on stages: Stage-4
+ Stage-2 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-5
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ $hdt$_1:src2
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ $hdt$_1:src2
+ TableScan
+ alias: src2
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ HashTable Sink Operator
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+
+ Stage: Stage-4
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: src1
+ Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col4
+ Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: UDFToInteger(_col0) (type: int), _col4 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.dest1
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.dest1
+
+ Stage: Stage-2
+ Stats-Aggr Operator
+
+PREHOOK: query: FROM srcpart src1 JOIN src src2 ON (src1.key = src2.key)
+INSERT OVERWRITE TABLE dest1 SELECT src1.key, src2.value
+where (src1.ds = '2008-04-08' or src1.ds = '2008-04-09' )and (src1.hr = '12' or src1.hr = '11')
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@srcpart
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+PREHOOK: Output: default@dest1
+POSTHOOK: query: FROM srcpart src1 JOIN src src2 ON (src1.key = src2.key)
+INSERT OVERWRITE TABLE dest1 SELECT src1.key, src2.value
+where (src1.ds = '2008-04-08' or src1.ds = '2008-04-09' )and (src1.hr = '12' or src1.hr = '11')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@srcpart
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+POSTHOOK: Output: default@dest1
+POSTHOOK: Lineage: dest1.key EXPRESSION [(srcpart)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: SELECT sum(hash(dest1.key,dest1.value)) FROM dest1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dest1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(dest1.key,dest1.value)) FROM dest1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dest1
+#### A masked pattern was here ####
+407444119660
http://git-wip-us.apache.org/repos/asf/hive/blob/8c8ff3f1/ql/src/test/results/clientpositive/constprog_semijoin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/constprog_semijoin.q.out b/ql/src/test/results/clientpositive/constprog_semijoin.q.out
index 0e0e883..940a148 100644
--- a/ql/src/test/results/clientpositive/constprog_semijoin.q.out
+++ b/ql/src/test/results/clientpositive/constprog_semijoin.q.out
@@ -502,7 +502,7 @@ STAGE PLANS:
alias: table1
Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((((dimid = 100) = true) and (dimid) IN (100, 200)) and (dimid = 100) is not null) (type: boolean)
+ predicate: (((dimid) IN (100, 200) and ((dimid = 100) = true)) and (dimid = 100) is not null) (type: boolean)
Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: id (type: int), val (type: string), val1 (type: string), dimid (type: int)
@@ -518,7 +518,7 @@ STAGE PLANS:
alias: table3
Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((((id = 100) = true) and (id) IN (100, 200)) and (id = 100) is not null) (type: boolean)
+ predicate: (((id) IN (100, 200) and ((id = 100) = true)) and (id = 100) is not null) (type: boolean)
Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: id (type: int), (id = 100) (type: boolean)
http://git-wip-us.apache.org/repos/asf/hive/blob/8c8ff3f1/ql/src/test/results/clientpositive/dynpart_sort_optimization_acid.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/dynpart_sort_optimization_acid.q.out b/ql/src/test/results/clientpositive/dynpart_sort_optimization_acid.q.out
index ddb05e2..eca29df 100644
--- a/ql/src/test/results/clientpositive/dynpart_sort_optimization_acid.q.out
+++ b/ql/src/test/results/clientpositive/dynpart_sort_optimization_acid.q.out
@@ -153,7 +153,7 @@ STAGE PLANS:
TableScan
alias: acid
Filter Operator
- predicate: ((key = 'foo') and (ds) IN ('2008-04-08')) (type: boolean)
+ predicate: (key = 'foo') (type: boolean)
Select Operator
expressions: ROW__ID (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>), ds (type: string)
outputColumnNames: _col0, _col3
@@ -390,7 +390,7 @@ STAGE PLANS:
TableScan
alias: acid
Filter Operator
- predicate: ((key = 'foo') and (ds) IN ('2008-04-08')) (type: boolean)
+ predicate: (key = 'foo') (type: boolean)
Select Operator
expressions: ROW__ID (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>), ds (type: string)
outputColumnNames: _col0, _col3
http://git-wip-us.apache.org/repos/asf/hive/blob/8c8ff3f1/ql/src/test/results/clientpositive/filter_in_or_dup.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/filter_in_or_dup.q.out b/ql/src/test/results/clientpositive/filter_in_or_dup.q.out
new file mode 100644
index 0000000..f863ac3
--- /dev/null
+++ b/ql/src/test/results/clientpositive/filter_in_or_dup.q.out
@@ -0,0 +1,96 @@
+PREHOOK: query: EXPLAIN
+SELECT f.key
+FROM cbo_t1 f
+WHERE (f.key = '1' OR f.key='2')
+AND f.key IN ('1', '2')
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN
+SELECT f.key
+FROM cbo_t1 f
+WHERE (f.key = '1' OR f.key='2')
+AND f.key IN ('1', '2')
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ TableScan
+ alias: f
+ Statistics: Num rows: 20 Data size: 262 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (key) IN ('1', '2') (type: boolean)
+ Statistics: Num rows: 10 Data size: 131 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 10 Data size: 131 Basic stats: COMPLETE Column stats: NONE
+ ListSink
+
+PREHOOK: query: EXPLAIN
+SELECT f.key
+FROM cbo_t1 f
+WHERE (f.key = '1' OR f.key = '2')
+AND f.key IN ('1', '2', '3')
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN
+SELECT f.key
+FROM cbo_t1 f
+WHERE (f.key = '1' OR f.key = '2')
+AND f.key IN ('1', '2', '3')
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ TableScan
+ alias: f
+ Statistics: Num rows: 20 Data size: 262 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (key) IN ('1', '2') (type: boolean)
+ Statistics: Num rows: 10 Data size: 131 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 10 Data size: 131 Basic stats: COMPLETE Column stats: NONE
+ ListSink
+
+PREHOOK: query: EXPLAIN
+SELECT f.key
+FROM cbo_t1 f
+WHERE (f.key = '1' OR f.key='2' OR f.key='3')
+AND f.key IN ('1', '2')
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN
+SELECT f.key
+FROM cbo_t1 f
+WHERE (f.key = '1' OR f.key='2' OR f.key='3')
+AND f.key IN ('1', '2')
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ TableScan
+ alias: f
+ Statistics: Num rows: 20 Data size: 262 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (key) IN ('1', '2') (type: boolean)
+ Statistics: Num rows: 10 Data size: 131 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 10 Data size: 131 Basic stats: COMPLETE Column stats: NONE
+ ListSink
+
http://git-wip-us.apache.org/repos/asf/hive/blob/8c8ff3f1/ql/src/test/results/clientpositive/perf/query13.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query13.q.out b/ql/src/test/results/clientpositive/perf/query13.q.out
index cc40e79..ad50576 100644
--- a/ql/src/test/results/clientpositive/perf/query13.q.out
+++ b/ql/src/test/results/clientpositive/perf/query13.q.out
@@ -128,7 +128,7 @@ Stage-0
SHUFFLE [RS_39]
Group By Operator [GBY_38] (rows=1 width=112)
Output:["_col0","_col1","_col2","_col3"],aggregations:["avg(_col5)","avg(_col7)","avg(_col8)","sum(_col8)"]
- Merge Join Operator [MERGEJOIN_73] (rows=18150000 width=1014)
+ Merge Join Operator [MERGEJOIN_73] (rows=9075000 width=1014)
Conds:RS_34._col0=RS_35._col0(Inner),Output:["_col5","_col7","_col8"]
<-Map 12 [SIMPLE_EDGE]
SHUFFLE [RS_35]
@@ -142,19 +142,19 @@ Stage-0
<-Reducer 5 [SIMPLE_EDGE]
SHUFFLE [RS_34]
PartitionCols:_col0
- Select Operator [SEL_30] (rows=16500000 width=1014)
+ Select Operator [SEL_30] (rows=8250000 width=1014)
Output:["_col0","_col5","_col7","_col8"]
- Filter Operator [FIL_29] (rows=16500000 width=1014)
+ Filter Operator [FIL_29] (rows=8250000 width=1014)
predicate:(((_col17) IN ('KY', 'GA', 'NM') and _col9 BETWEEN 100 AND 200) or ((_col17) IN ('MT', 'OR', 'IN') and _col9 BETWEEN 150 AND 300) or ((_col17) IN ('WI', 'MO', 'WV') and _col9 BETWEEN 50 AND 250))
- Merge Join Operator [MERGEJOIN_72] (rows=22000000 width=1014)
+ Merge Join Operator [MERGEJOIN_72] (rows=11000000 width=1014)
Conds:RS_26._col3=RS_27._col0(Inner),Output:["_col0","_col5","_col7","_col8","_col9","_col17"]
<-Map 11 [SIMPLE_EDGE]
SHUFFLE [RS_27]
PartitionCols:_col0
- Select Operator [SEL_25] (rows=20000000 width=1014)
+ Select Operator [SEL_25] (rows=10000000 width=1014)
Output:["_col0","_col1"]
- Filter Operator [FIL_67] (rows=20000000 width=1014)
- predicate:((((ca_state) IN ('KY', 'GA', 'NM') or (ca_state) IN ('MT', 'OR', 'IN') or (ca_state) IN ('WI', 'MO', 'WV')) and (ca_country = 'United States')) and ca_address_sk is not null)
+ Filter Operator [FIL_67] (rows=10000000 width=1014)
+ predicate:(((ca_state) IN ('KY', 'GA', 'NM', 'MT', 'OR', 'IN', 'WI', 'MO', 'WV') and (ca_country = 'United States')) and ca_address_sk is not null)
TableScan [TS_23] (rows=40000000 width=1014)
default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state","ca_country"]
<-Reducer 4 [SIMPLE_EDGE]
http://git-wip-us.apache.org/repos/asf/hive/blob/8c8ff3f1/ql/src/test/results/clientpositive/perf/query27.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query27.q.out b/ql/src/test/results/clientpositive/perf/query27.q.out
index 635c402..3a32d7b 100644
--- a/ql/src/test/results/clientpositive/perf/query27.q.out
+++ b/ql/src/test/results/clientpositive/perf/query27.q.out
@@ -57,7 +57,7 @@ Stage-0
Select Operator [SEL_11] (rows=852 width=1910)
Output:["_col0","_col1"]
Filter Operator [FIL_53] (rows=852 width=1910)
- predicate:((s_state) IN ('KS', 'AL', 'MN', 'AL', 'SC', 'VT') and s_store_sk is not null)
+ predicate:((s_state) IN ('KS', 'AL', 'MN', 'SC', 'VT') and s_store_sk is not null)
TableScan [TS_9] (rows=1704 width=1910)
default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_state"]
<-Reducer 3 [SIMPLE_EDGE]
http://git-wip-us.apache.org/repos/asf/hive/blob/8c8ff3f1/ql/src/test/results/clientpositive/perf/query34.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query34.q.out b/ql/src/test/results/clientpositive/perf/query34.q.out
index 6fa6985..a08c3ff 100644
--- a/ql/src/test/results/clientpositive/perf/query34.q.out
+++ b/ql/src/test/results/clientpositive/perf/query34.q.out
@@ -94,7 +94,7 @@ Stage-0
Select Operator [SEL_5] (rows=36524 width=1119)
Output:["_col0"]
Filter Operator [FIL_53] (rows=36524 width=1119)
- predicate:(((d_dom BETWEEN 1 AND 3 or d_dom BETWEEN 25 AND 28) and (d_year) IN (1998, 1999, 2000)) and d_date_sk is not null)
+ predicate:(((d_year) IN (1998, 1999, 2000) and (d_dom BETWEEN 1 AND 3 or d_dom BETWEEN 25 AND 28)) and d_date_sk is not null)
TableScan [TS_3] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_dom"]
http://git-wip-us.apache.org/repos/asf/hive/blob/8c8ff3f1/ql/src/test/results/clientpositive/perf/query48.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query48.q.out b/ql/src/test/results/clientpositive/perf/query48.q.out
index 691f5ad..d536bb5 100644
--- a/ql/src/test/results/clientpositive/perf/query48.q.out
+++ b/ql/src/test/results/clientpositive/perf/query48.q.out
@@ -23,7 +23,7 @@ Stage-0
SHUFFLE [RS_31]
Group By Operator [GBY_30] (rows=1 width=8)
Output:["_col0"],aggregations:["sum(_col4)"]
- Merge Join Operator [MERGEJOIN_57] (rows=18150000 width=1014)
+ Merge Join Operator [MERGEJOIN_57] (rows=9075000 width=1014)
Conds:RS_26._col0=RS_27._col0(Inner),Output:["_col4"]
<-Map 10 [SIMPLE_EDGE]
SHUFFLE [RS_27]
@@ -37,19 +37,19 @@ Stage-0
<-Reducer 4 [SIMPLE_EDGE]
SHUFFLE [RS_26]
PartitionCols:_col0
- Select Operator [SEL_22] (rows=16500000 width=1014)
+ Select Operator [SEL_22] (rows=8250000 width=1014)
Output:["_col0","_col4"]
- Filter Operator [FIL_21] (rows=16500000 width=1014)
+ Filter Operator [FIL_21] (rows=8250000 width=1014)
predicate:(((_col12) IN ('KY', 'GA', 'NM') and _col6 BETWEEN 0 AND 2000) or ((_col12) IN ('MT', 'OR', 'IN') and _col6 BETWEEN 150 AND 3000) or ((_col12) IN ('WI', 'MO', 'WV') and _col6 BETWEEN 50 AND 25000))
- Merge Join Operator [MERGEJOIN_56] (rows=22000000 width=1014)
+ Merge Join Operator [MERGEJOIN_56] (rows=11000000 width=1014)
Conds:RS_18._col2=RS_19._col0(Inner),Output:["_col0","_col4","_col6","_col12"]
<-Map 9 [SIMPLE_EDGE]
SHUFFLE [RS_19]
PartitionCols:_col0
- Select Operator [SEL_11] (rows=20000000 width=1014)
+ Select Operator [SEL_11] (rows=10000000 width=1014)
Output:["_col0","_col1"]
- Filter Operator [FIL_52] (rows=20000000 width=1014)
- predicate:((((ca_state) IN ('KY', 'GA', 'NM') or (ca_state) IN ('MT', 'OR', 'IN') or (ca_state) IN ('WI', 'MO', 'WV')) and (ca_country = 'United States')) and ca_address_sk is not null)
+ Filter Operator [FIL_52] (rows=10000000 width=1014)
+ predicate:(((ca_state) IN ('KY', 'GA', 'NM', 'MT', 'OR', 'IN', 'WI', 'MO', 'WV') and (ca_country = 'United States')) and ca_address_sk is not null)
TableScan [TS_9] (rows=40000000 width=1014)
default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state","ca_country"]
<-Reducer 3 [SIMPLE_EDGE]
http://git-wip-us.apache.org/repos/asf/hive/blob/8c8ff3f1/ql/src/test/results/clientpositive/perf/query68.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query68.q.out b/ql/src/test/results/clientpositive/perf/query68.q.out
index 7828cfc..38e4644 100644
--- a/ql/src/test/results/clientpositive/perf/query68.q.out
+++ b/ql/src/test/results/clientpositive/perf/query68.q.out
@@ -128,7 +128,7 @@ Stage-0
Select Operator [SEL_5] (rows=18262 width=1119)
Output:["_col0"]
Filter Operator [FIL_79] (rows=18262 width=1119)
- predicate:((d_dom BETWEEN 1 AND 2 and (d_year) IN (1998, 1999, 2000)) and d_date_sk is not null)
+ predicate:(((d_year) IN (1998, 1999, 2000) and d_dom BETWEEN 1 AND 2) and d_date_sk is not null)
TableScan [TS_3] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_dom"]
http://git-wip-us.apache.org/repos/asf/hive/blob/8c8ff3f1/ql/src/test/results/clientpositive/perf/query73.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query73.q.out b/ql/src/test/results/clientpositive/perf/query73.q.out
index e367f51..cf3a75e 100644
--- a/ql/src/test/results/clientpositive/perf/query73.q.out
+++ b/ql/src/test/results/clientpositive/perf/query73.q.out
@@ -94,7 +94,7 @@ Stage-0
Select Operator [SEL_5] (rows=18262 width=1119)
Output:["_col0"]
Filter Operator [FIL_53] (rows=18262 width=1119)
- predicate:((d_dom BETWEEN 1 AND 2 and (d_year) IN (1998, 1999, 2000)) and d_date_sk is not null)
+ predicate:(((d_year) IN (1998, 1999, 2000) and d_dom BETWEEN 1 AND 2) and d_date_sk is not null)
TableScan [TS_3] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_dom"]
http://git-wip-us.apache.org/repos/asf/hive/blob/8c8ff3f1/ql/src/test/results/clientpositive/perf/query79.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query79.q.out b/ql/src/test/results/clientpositive/perf/query79.q.out
index fdc5773..bf537b9 100644
--- a/ql/src/test/results/clientpositive/perf/query79.q.out
+++ b/ql/src/test/results/clientpositive/perf/query79.q.out
@@ -96,7 +96,7 @@ Stage-0
Select Operator [SEL_5] (rows=18262 width=1119)
Output:["_col0"]
Filter Operator [FIL_53] (rows=18262 width=1119)
- predicate:(((d_dow = 1) and (d_year) IN (1998, 1999, 2000)) and d_date_sk is not null)
+ predicate:(((d_year) IN (1998, 1999, 2000) and (d_dow = 1)) and d_date_sk is not null)
TableScan [TS_3] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_dow"]
http://git-wip-us.apache.org/repos/asf/hive/blob/8c8ff3f1/ql/src/test/results/clientpositive/perf/query82.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query82.q.out b/ql/src/test/results/clientpositive/perf/query82.q.out
index 2461644..57a50c7 100644
--- a/ql/src/test/results/clientpositive/perf/query82.q.out
+++ b/ql/src/test/results/clientpositive/perf/query82.q.out
@@ -51,7 +51,7 @@ Stage-0
Select Operator [SEL_2] (rows=115500 width=1436)
Output:["_col0","_col1","_col2","_col3"]
Filter Operator [FIL_38] (rows=115500 width=1436)
- predicate:((i_current_price BETWEEN 30 AND 60 and (i_manufact_id) IN (437, 129, 727, 663)) and i_item_sk is not null)
+ predicate:(((i_manufact_id) IN (437, 129, 727, 663) and i_current_price BETWEEN 30 AND 60) and i_item_sk is not null)
TableScan [TS_0] (rows=462000 width=1436)
default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id","i_item_desc","i_current_price","i_manufact_id"]
<-Map 6 [SIMPLE_EDGE]
http://git-wip-us.apache.org/repos/asf/hive/blob/8c8ff3f1/ql/src/test/results/clientpositive/perf/query85.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query85.q.out b/ql/src/test/results/clientpositive/perf/query85.q.out
index 72ac500..93b5f4e 100644
--- a/ql/src/test/results/clientpositive/perf/query85.q.out
+++ b/ql/src/test/results/clientpositive/perf/query85.q.out
@@ -23,22 +23,22 @@ Stage-0
File Output Operator [FS_57]
Limit [LIM_56] (rows=100 width=1014)
Number of rows:100
- Select Operator [SEL_55] (rows=9982500 width=1014)
+ Select Operator [SEL_55] (rows=4991250 width=1014)
Output:["_col0","_col1","_col2","_col3"]
<-Reducer 9 [SIMPLE_EDGE]
SHUFFLE [RS_54]
- Select Operator [SEL_53] (rows=9982500 width=1014)
+ Select Operator [SEL_53] (rows=4991250 width=1014)
Output:["_col0","_col1","_col2","_col3"]
- Group By Operator [GBY_52] (rows=9982500 width=1014)
+ Group By Operator [GBY_52] (rows=4991250 width=1014)
Output:["_col0","_col1","_col2","_col3"],aggregations:["avg(VALUE._col0)","avg(VALUE._col1)","avg(VALUE._col2)"],keys:KEY._col0
<-Reducer 8 [SIMPLE_EDGE]
SHUFFLE [RS_51]
PartitionCols:_col0
- Group By Operator [GBY_50] (rows=19965000 width=1014)
+ Group By Operator [GBY_50] (rows=9982500 width=1014)
Output:["_col0","_col1","_col2","_col3"],aggregations:["avg(_col4)","avg(_col14)","avg(_col13)"],keys:_col28
- Select Operator [SEL_49] (rows=19965000 width=1014)
+ Select Operator [SEL_49] (rows=9982500 width=1014)
Output:["_col28","_col4","_col14","_col13"]
- Merge Join Operator [MERGEJOIN_107] (rows=19965000 width=1014)
+ Merge Join Operator [MERGEJOIN_107] (rows=9982500 width=1014)
Conds:RS_46._col11=RS_47._col0(Inner),Output:["_col4","_col13","_col14","_col28"]
<-Map 17 [SIMPLE_EDGE]
SHUFFLE [RS_47]
@@ -52,7 +52,7 @@ Stage-0
<-Reducer 7 [SIMPLE_EDGE]
SHUFFLE [RS_46]
PartitionCols:_col11
- Merge Join Operator [MERGEJOIN_106] (rows=18150000 width=1014)
+ Merge Join Operator [MERGEJOIN_106] (rows=9075000 width=1014)
Conds:RS_43._col0=RS_44._col0(Inner),Output:["_col4","_col11","_col13","_col14"]
<-Map 16 [SIMPLE_EDGE]
SHUFFLE [RS_44]
@@ -66,19 +66,19 @@ Stage-0
<-Reducer 6 [SIMPLE_EDGE]
SHUFFLE [RS_43]
PartitionCols:_col0
- Select Operator [SEL_36] (rows=16500000 width=1014)
+ Select Operator [SEL_36] (rows=8250000 width=1014)
Output:["_col0","_col11","_col13","_col14","_col4"]
- Filter Operator [FIL_35] (rows=16500000 width=1014)
+ Filter Operator [FIL_35] (rows=8250000 width=1014)
predicate:(((_col23) IN ('KY', 'GA', 'NM') and _col6 BETWEEN 100 AND 200) or ((_col23) IN ('MT', 'OR', 'IN') and _col6 BETWEEN 150 AND 300) or ((_col23) IN ('WI', 'MO', 'WV') and _col6 BETWEEN 50 AND 250))
- Merge Join Operator [MERGEJOIN_105] (rows=22000000 width=1014)
+ Merge Join Operator [MERGEJOIN_105] (rows=11000000 width=1014)
Conds:RS_32._col9=RS_33._col0(Inner),Output:["_col0","_col4","_col6","_col11","_col13","_col14","_col23"]
<-Map 15 [SIMPLE_EDGE]
SHUFFLE [RS_33]
PartitionCols:_col0
- Select Operator [SEL_28] (rows=20000000 width=1014)
+ Select Operator [SEL_28] (rows=10000000 width=1014)
Output:["_col0","_col1"]
- Filter Operator [FIL_98] (rows=20000000 width=1014)
- predicate:((((ca_state) IN ('KY', 'GA', 'NM') or (ca_state) IN ('MT', 'OR', 'IN') or (ca_state) IN ('WI', 'MO', 'WV')) and (ca_country = 'United States')) and ca_address_sk is not null)
+ Filter Operator [FIL_98] (rows=10000000 width=1014)
+ predicate:(((ca_state) IN ('KY', 'GA', 'NM', 'MT', 'OR', 'IN', 'WI', 'MO', 'WV') and (ca_country = 'United States')) and ca_address_sk is not null)
TableScan [TS_26] (rows=40000000 width=1014)
default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state","ca_country"]
<-Reducer 5 [SIMPLE_EDGE]
http://git-wip-us.apache.org/repos/asf/hive/blob/8c8ff3f1/ql/src/test/results/clientpositive/pointlookup2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/pointlookup2.q.out b/ql/src/test/results/clientpositive/pointlookup2.q.out
index fb17e72..869e4cd 100644
--- a/ql/src/test/results/clientpositive/pointlookup2.q.out
+++ b/ql/src/test/results/clientpositive/pointlookup2.q.out
@@ -985,21 +985,17 @@ STAGE PLANS:
alias: t1
Statistics: Num rows: 40 Data size: 320 Basic stats: COMPLETE Column stats: NONE
GatherStats: false
- Filter Operator
- isSamplingPred: false
- predicate: (ds) IN ('2000-04-08', '2000-04-09') (type: boolean)
- Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: key (type: int), value (type: string), ds (type: string)
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- null sort order:
- sort order:
- Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: NONE
- tag: 0
- value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string)
- auto parallelism: false
+ Select Operator
+ expressions: key (type: int), value (type: string), ds (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 40 Data size: 320 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ null sort order:
+ sort order:
+ Statistics: Num rows: 40 Data size: 320 Basic stats: COMPLETE Column stats: NONE
+ tag: 0
+ value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string)
+ auto parallelism: false
TableScan
alias: t2
Statistics: Num rows: 1 Data size: 18 Basic stats: COMPLETE Column stats: NONE
@@ -1169,11 +1165,11 @@ STAGE PLANS:
0
1
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 44 Data size: 352 Basic stats: COMPLETE Column stats: NONE
Filter Operator
isSamplingPred: false
- predicate: (struct(_col4,_col2)) IN (const struct(1,'2000-04-08'), const struct(2,'2000-04-09')) (type: boolean)
- Statistics: Num rows: 11 Data size: 88 Basic stats: COMPLETE Column stats: NONE
+ predicate: (struct(_col2,_col4)) IN (const struct('2000-04-08',1), const struct('2000-04-09',2)) (type: boolean)
+ Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
GlobalTableId: 0
@@ -1201,7 +1197,7 @@ STAGE PLANS:
key expressions: _col4 (type: int), _col5 (type: string), _col2 (type: string)
null sort order: aaa
sort order: +++
- Statistics: Num rows: 11 Data size: 88 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE
tag: -1
value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string)
auto parallelism: false
@@ -1235,13 +1231,13 @@ STAGE PLANS:
Select Operator
expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY.reducesinkkey2 (type: string), VALUE._col2 (type: string), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 11 Data size: 88 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
GlobalTableId: 0
#### A masked pattern was here ####
NumFilesPerFileSink: 1
- Statistics: Num rows: 11 Data size: 88 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE
#### A masked pattern was here ####
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/8c8ff3f1/ql/src/test/results/clientpositive/pointlookup3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/pointlookup3.q.out b/ql/src/test/results/clientpositive/pointlookup3.q.out
index d5c4157..e98ba76 100644
--- a/ql/src/test/results/clientpositive/pointlookup3.q.out
+++ b/ql/src/test/results/clientpositive/pointlookup3.q.out
@@ -129,7 +129,7 @@ STAGE PLANS:
GatherStats: false
Filter Operator
isSamplingPred: false
- predicate: (struct(ds1,key)) IN (const struct('2000-04-08',1), const struct('2000-04-09',2)) (type: boolean)
+ predicate: (struct(key,ds1)) IN (const struct(1,'2000-04-08'), const struct(2,'2000-04-09')) (type: boolean)
Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: int), value (type: string), ds1 (type: string), ds2 (type: string)
@@ -374,14 +374,14 @@ STAGE PLANS:
GatherStats: false
Filter Operator
isSamplingPred: false
- predicate: (key = 1) (type: boolean)
+ predicate: (struct(key,ds1)) IN (const struct(1,'2000-04-08'), const struct(2,'2000-04-09')) (type: boolean)
Statistics: Num rows: 10 Data size: 80 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: value (type: string), ds1 (type: string)
- outputColumnNames: _col1, _col2
+ expressions: key (type: int), value (type: string), ds1 (type: string)
+ outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 10 Data size: 80 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: 1 (type: int), _col1 (type: string), _col2 (type: string), '2001-04-08' (type: string)
+ key expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), '2001-04-08' (type: string)
null sort order: aaaa
sort order: ++++
Statistics: Num rows: 10 Data size: 80 Basic stats: COMPLETE Column stats: NONE
@@ -441,7 +441,7 @@ STAGE PLANS:
Needs Tagging: false
Reduce Operator Tree:
Select Operator
- expressions: 1 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), '2001-04-08' (type: string)
+ expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), '2001-04-08' (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 10 Data size: 80 Basic stats: COMPLETE Column stats: NONE
File Output Operator
@@ -1149,21 +1149,17 @@ STAGE PLANS:
alias: t1
Statistics: Num rows: 40 Data size: 320 Basic stats: COMPLETE Column stats: NONE
GatherStats: false
- Filter Operator
- isSamplingPred: false
- predicate: (ds1) IN ('2000-04-08', '2000-04-09') (type: boolean)
- Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: key (type: int), value (type: string), ds1 (type: string), ds2 (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- null sort order:
- sort order:
- Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: NONE
- tag: 0
- value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string)
- auto parallelism: false
+ Select Operator
+ expressions: key (type: int), value (type: string), ds1 (type: string), ds2 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 40 Data size: 320 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ null sort order:
+ sort order:
+ Statistics: Num rows: 40 Data size: 320 Basic stats: COMPLETE Column stats: NONE
+ tag: 0
+ value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string)
+ auto parallelism: false
TableScan
alias: t1
Statistics: Num rows: 60 Data size: 480 Basic stats: COMPLETE Column stats: NONE
@@ -1337,11 +1333,11 @@ STAGE PLANS:
0
1
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 33 Data size: 264 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 44 Data size: 352 Basic stats: COMPLETE Column stats: NONE
Filter Operator
isSamplingPred: false
- predicate: (struct(_col4,_col2)) IN (const struct(1,'2000-04-08'), const struct(2,'2000-04-09')) (type: boolean)
- Statistics: Num rows: 16 Data size: 128 Basic stats: COMPLETE Column stats: NONE
+ predicate: (struct(_col2,_col4)) IN (const struct('2000-04-08',1), const struct('2000-04-09',2)) (type: boolean)
+ Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
GlobalTableId: 0
@@ -1369,7 +1365,7 @@ STAGE PLANS:
key expressions: _col4 (type: int), _col5 (type: string), _col2 (type: string)
null sort order: aaa
sort order: +++
- Statistics: Num rows: 16 Data size: 128 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE
tag: -1
value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string), _col6 (type: string), _col7 (type: string)
auto parallelism: false
@@ -1403,13 +1399,13 @@ STAGE PLANS:
Select Operator
expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY.reducesinkkey2 (type: string), VALUE._col2 (type: string), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string), VALUE._col3 (type: string), VALUE._col4 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 16 Data size: 128 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
GlobalTableId: 0
#### A masked pattern was here ####
NumFilesPerFileSink: 1
- Statistics: Num rows: 16 Data size: 128 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE
#### A masked pattern was here ####
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/8c8ff3f1/ql/src/test/results/clientpositive/pointlookup4.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/pointlookup4.q.out b/ql/src/test/results/clientpositive/pointlookup4.q.out
index 0a9bd3e..6236272 100644
--- a/ql/src/test/results/clientpositive/pointlookup4.q.out
+++ b/ql/src/test/results/clientpositive/pointlookup4.q.out
@@ -384,7 +384,7 @@ STAGE PLANS:
GatherStats: false
Filter Operator
isSamplingPred: false
- predicate: (struct(ds1,key,ds2)) IN (const struct('2000-04-08',1,'2001-04-08'), const struct('2000-04-09',2,'2001-04-09')) (type: boolean)
+ predicate: (struct(key,ds1,ds2)) IN (const struct(1,'2000-04-08','2001-04-08'), const struct(2,'2000-04-09','2001-04-09')) (type: boolean)
Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: int), value (type: string), ds1 (type: string), ds2 (type: string)
http://git-wip-us.apache.org/repos/asf/hive/blob/8c8ff3f1/ql/src/test/results/clientpositive/spark/constprog_semijoin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/constprog_semijoin.q.out b/ql/src/test/results/clientpositive/spark/constprog_semijoin.q.out
index 2547405..0ab1365 100644
--- a/ql/src/test/results/clientpositive/spark/constprog_semijoin.q.out
+++ b/ql/src/test/results/clientpositive/spark/constprog_semijoin.q.out
@@ -523,7 +523,7 @@ STAGE PLANS:
alias: table1
Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((((dimid = 100) = true) and (dimid) IN (100, 200)) and (dimid = 100) is not null) (type: boolean)
+ predicate: (((dimid) IN (100, 200) and ((dimid = 100) = true)) and (dimid = 100) is not null) (type: boolean)
Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: id (type: int), val (type: string), val1 (type: string), dimid (type: int)
@@ -541,7 +541,7 @@ STAGE PLANS:
alias: table3
Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((((id = 100) = true) and (id) IN (100, 200)) and (id = 100) is not null) (type: boolean)
+ predicate: (((id) IN (100, 200) and ((id = 100) = true)) and (id = 100) is not null) (type: boolean)
Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: id (type: int), (id = 100) (type: boolean)
http://git-wip-us.apache.org/repos/asf/hive/blob/8c8ff3f1/ql/src/test/results/clientpositive/tez/bucketpruning1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/bucketpruning1.q.out b/ql/src/test/results/clientpositive/tez/bucketpruning1.q.out
index 5315f2c..3557a3b 100644
--- a/ql/src/test/results/clientpositive/tez/bucketpruning1.q.out
+++ b/ql/src/test/results/clientpositive/tez/bucketpruning1.q.out
@@ -1011,13 +1011,13 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: srcbucket_pruned
- filterExpr: (((value = 'One') and (key) IN (2, 3)) and (ds = '2008-04-08')) (type: boolean)
+ filterExpr: (((key) IN (2, 3) and (value = 'One')) and (ds = '2008-04-08')) (type: boolean)
buckets included: [2,3,] of 16
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
GatherStats: false
Filter Operator
isSamplingPred: false
- predicate: (((value = 'One') and (key) IN (2, 3)) and (ds = '2008-04-08')) (type: boolean)
+ predicate: (((key) IN (2, 3) and (value = 'One')) and (ds = '2008-04-08')) (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: int), 'One' (type: string), '2008-04-08' (type: string)
@@ -1700,12 +1700,12 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: srcbucket_pruned
- filterExpr: (((value = 'One') and (key) IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17)) and (ds = '2008-04-08')) (type: boolean)
+ filterExpr: (((key) IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17) and (value = 'One')) and (ds = '2008-04-08')) (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
GatherStats: false
Filter Operator
isSamplingPred: false
- predicate: (((value = 'One') and (key) IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17)) and (ds = '2008-04-08')) (type: boolean)
+ predicate: (((key) IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17) and (value = 'One')) and (ds = '2008-04-08')) (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: int), 'One' (type: string), '2008-04-08' (type: string)
http://git-wip-us.apache.org/repos/asf/hive/blob/8c8ff3f1/ql/src/test/results/clientpositive/tez/constprog_semijoin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/constprog_semijoin.q.out b/ql/src/test/results/clientpositive/tez/constprog_semijoin.q.out
index 7a9932a..8fecbd7 100644
--- a/ql/src/test/results/clientpositive/tez/constprog_semijoin.q.out
+++ b/ql/src/test/results/clientpositive/tez/constprog_semijoin.q.out
@@ -317,7 +317,7 @@ Stage-0
Select Operator [SEL_2] (rows=2 width=20)
Output:["_col0","_col1","_col2","_col3"]
Filter Operator [FIL_15] (rows=2 width=20)
- predicate:((((dimid = 100) = true) and (dimid) IN (100, 200)) and (dimid = 100) is not null)
+ predicate:(((dimid) IN (100, 200) and ((dimid = 100) = true)) and (dimid = 100) is not null)
TableScan [TS_0] (rows=10 width=20)
default@table1,table1,Tbl:COMPLETE,Col:NONE,Output:["id","val","val1","dimid"]
<-Map 3 [SIMPLE_EDGE]
@@ -328,7 +328,7 @@ Stage-0
Select Operator [SEL_5] (rows=1 width=3)
Output:["_col0","_col1"]
Filter Operator [FIL_17] (rows=1 width=3)
- predicate:((((id = 100) = true) and (id) IN (100, 200)) and (id = 100) is not null)
+ predicate:(((id) IN (100, 200) and ((id = 100) = true)) and (id = 100) is not null)
TableScan [TS_3] (rows=5 width=3)
default@table3,table3,Tbl:COMPLETE,Col:NONE,Output:["id"]
http://git-wip-us.apache.org/repos/asf/hive/blob/8c8ff3f1/ql/src/test/results/clientpositive/tez/vector_mr_diff_schema_alias.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_mr_diff_schema_alias.q.out b/ql/src/test/results/clientpositive/tez/vector_mr_diff_schema_alias.q.out
index 0d6ad69..5a2ab91 100644
--- a/ql/src/test/results/clientpositive/tez/vector_mr_diff_schema_alias.q.out
+++ b/ql/src/test/results/clientpositive/tez/vector_mr_diff_schema_alias.q.out
@@ -278,7 +278,7 @@ STAGE PLANS:
alias: store
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
- predicate: ((s_state) IN ('KS', 'AL', 'MN', 'AL', 'SC', 'VT') and s_store_sk is not null) (type: boolean)
+ predicate: ((s_state) IN ('KS', 'AL', 'MN', 'SC', 'VT') and s_store_sk is not null) (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: s_store_sk (type: int), s_state (type: string)
http://git-wip-us.apache.org/repos/asf/hive/blob/8c8ff3f1/ql/src/test/results/clientpositive/vector_mr_diff_schema_alias.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_mr_diff_schema_alias.q.out b/ql/src/test/results/clientpositive/vector_mr_diff_schema_alias.q.out
index 9fce991..a9e25e1 100644
--- a/ql/src/test/results/clientpositive/vector_mr_diff_schema_alias.q.out
+++ b/ql/src/test/results/clientpositive/vector_mr_diff_schema_alias.q.out
@@ -269,7 +269,7 @@ STAGE PLANS:
alias: store
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
- predicate: ((s_state) IN ('KS', 'AL', 'MN', 'AL', 'SC', 'VT') and s_store_sk is not null) (type: boolean)
+ predicate: ((s_state) IN ('KS', 'AL', 'MN', 'SC', 'VT') and s_store_sk is not null) (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: s_store_sk (type: int), s_state (type: string)
[48/50] [abbrv] hive git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/hive
Posted by jd...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/hive
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/51efcb80
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/51efcb80
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/51efcb80
Branch: refs/heads/llap
Commit: 51efcb80e1d09e490dab644bcad2cab54c99e353
Parents: 39d66a4 8c1f055
Author: Dmitry Tolpeko <dm...@gmail.com>
Authored: Wed Mar 30 00:23:51 2016 -0700
Committer: Dmitry Tolpeko <dm...@gmail.com>
Committed: Wed Mar 30 00:23:51 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/ant/GenVectorCode.java | 531 ++++-----
.../apache/hadoop/hive/common/FileUtils.java | 54 +
.../hive/common/type/HiveIntervalDayTime.java | 245 ----
.../org/apache/hadoop/hive/conf/HiveConf.java | 15 +-
.../org/apache/hive/common/util/DateUtils.java | 19 -
.../hive/contrib/serde2/MultiDelimitSerDe.java | 23 +-
data/files/alltypesorc3xcols | Bin 0 -> 1504592 bytes
data/files/timestamps.txt | 50 +
.../hive/hcatalog/streaming/HiveEndPoint.java | 11 +
.../hcatalog/templeton/SecureProxySupport.java | 46 +-
.../listener/TestDbNotificationListener.java | 18 +
.../org/apache/hive/jdbc/miniHS2/MiniHS2.java | 56 +-
.../hive/metastore/TestHiveMetaStore.java | 20 +-
.../jdbc/TestJdbcWithLocalClusterSpark.java | 2 +-
.../apache/hive/jdbc/TestJdbcWithMiniMr.java | 2 +-
...stMultiSessionsHS2WithLocalClusterSpark.java | 6 +-
.../jdbc/TestServiceDiscoveryWithMiniHS2.java | 132 +++
.../jdbc/authorization/TestHS2AuthzContext.java | 4 +-
.../authorization/TestJdbcMetadataApiAuth.java | 2 +-
.../TestJdbcWithSQLAuthorization.java | 2 +-
.../test/resources/testconfiguration.properties | 5 +
.../hive/jdbc/ZooKeeperHiveClientHelper.java | 21 +-
.../hadoop/hive/llap/io/api/LlapProxy.java | 2 +
.../org/apache/hadoop/hive/llap/LlapUtil.java | 26 +
.../apache/hadoop/hive/llap/tez/Converters.java | 1 +
.../hadoop/hive/llap/tez/TestConverters.java | 190 +++
.../llap/IncrementalObjectSizeEstimator.java | 54 +-
.../hadoop/hive/llap/cache/LlapDataBuffer.java | 12 +-
.../hive/llap/cache/LowLevelCacheImpl.java | 35 +-
.../llap/cache/LowLevelCacheMemoryManager.java | 6 +-
.../llap/cache/LowLevelFifoCachePolicy.java | 4 +-
.../llap/cache/LowLevelLrfuCachePolicy.java | 14 +-
.../hive/llap/cache/SimpleBufferManager.java | 8 +-
.../hive/llap/daemon/impl/LlapDaemon.java | 6 +-
.../hive/llap/io/api/impl/LlapInputFormat.java | 32 +-
.../hive/llap/io/api/impl/LlapIoImpl.java | 21 +-
.../llap/io/decode/OrcColumnVectorProducer.java | 4 +-
.../llap/io/encoded/OrcEncodedDataReader.java | 95 +-
.../hadoop/hive/llap/old/BufferInProgress.java | 82 --
.../apache/hadoop/hive/llap/old/BufferPool.java | 225 ----
.../hadoop/hive/llap/old/CachePolicy.java | 34 -
.../apache/hadoop/hive/llap/old/ChunkPool.java | 237 ----
.../resources/llap-daemon-log4j2.properties | 14 +-
.../hive/metastore/MetaStoreDirectSql.java | 32 +-
.../hadoop/hive/metastore/ObjectStore.java | 2 +-
.../hive/metastore/StatObjectConverter.java | 2 +-
.../hadoop/hive/metastore/hbase/HBaseUtils.java | 8 +-
.../hadoop/hive/metastore/hbase/StatsCache.java | 20 +-
.../stats/BinaryColumnStatsAggregator.java | 43 +-
.../stats/BooleanColumnStatsAggregator.java | 42 +-
.../hbase/stats/ColumnStatsAggregator.java | 12 +-
.../stats/ColumnStatsAggregatorFactory.java | 8 +-
.../stats/DecimalColumnStatsAggregator.java | 340 +++++-
.../stats/DoubleColumnStatsAggregator.java | 307 ++++-
.../hbase/stats/IExtrapolatePartStatus.java | 30 +
.../hbase/stats/LongColumnStatsAggregator.java | 305 ++++-
.../stats/StringColumnStatsAggregator.java | 85 +-
...stHBaseAggregateStatsCacheWithBitVector.java | 6 +-
.../TestHBaseAggregateStatsExtrapolation.java | 717 ++++++++++++
.../TestHBaseAggregateStatsNDVUniformDist.java | 581 ++++++++++
orc/src/java/org/apache/orc/OrcFile.java | 21 +-
.../java/org/apache/orc/impl/WriterImpl.java | 41 +-
...eColumnArithmeticIntervalYearMonthColumn.txt | 56 +-
...eColumnArithmeticIntervalYearMonthScalar.txt | 55 +-
.../DateColumnArithmeticTimestampColumn.txt | 141 ++-
.../DateColumnArithmeticTimestampColumnBase.txt | 171 ---
.../DateColumnArithmeticTimestampScalar.txt | 113 +-
.../DateColumnArithmeticTimestampScalarBase.txt | 137 ---
...eScalarArithmeticIntervalYearMonthColumn.txt | 53 +-
.../DateScalarArithmeticTimestampColumn.txt | 108 +-
.../DateScalarArithmeticTimestampColumnBase.txt | 147 ---
...ayTimeColumnCompareIntervalDayTimeColumn.txt | 52 -
...ayTimeColumnCompareIntervalDayTimeScalar.txt | 55 -
...ayTimeScalarCompareIntervalDayTimeColumn.txt | 55 -
...erLongDoubleColumnCompareTimestampColumn.txt | 2 +-
...erLongDoubleColumnCompareTimestampScalar.txt | 4 +-
...erLongDoubleScalarCompareTimestampColumn.txt | 4 +
.../FilterTimestampColumnBetween.txt | 11 +-
...terTimestampColumnCompareTimestampColumn.txt | 417 ++++++-
...imestampColumnCompareTimestampColumnBase.txt | 429 -------
...terTimestampColumnCompareTimestampScalar.txt | 128 ++-
...imestampColumnCompareTimestampScalarBase.txt | 145 ---
...erTimestampScalarCompareLongDoubleColumn.txt | 3 +-
...terTimestampScalarCompareTimestampColumn.txt | 132 ++-
...imestampScalarCompareTimestampColumnBase.txt | 147 ---
...ayTimeColumnCompareIntervalDayTimeColumn.txt | 54 -
...ayTimeColumnCompareIntervalDayTimeScalar.txt | 57 -
...ayTimeScalarCompareIntervalDayTimeColumn.txt | 57 -
...ervalYearMonthColumnArithmeticDateColumn.txt | 55 +-
...ervalYearMonthColumnArithmeticDateScalar.txt | 51 +-
...YearMonthColumnArithmeticTimestampColumn.txt | 63 +-
...YearMonthColumnArithmeticTimestampScalar.txt | 48 +-
...ervalYearMonthScalarArithmeticDateColumn.txt | 51 +-
...YearMonthScalarArithmeticTimestampColumn.txt | 55 +-
.../LongDoubleColumnCompareTimestampColumn.txt | 1 -
.../LongDoubleColumnCompareTimestampScalar.txt | 3 +-
.../LongDoubleScalarCompareTimestampColumn.txt | 1 +
.../TimestampColumnArithmeticDateColumn.txt | 138 ++-
.../TimestampColumnArithmeticDateColumnBase.txt | 172 ---
.../TimestampColumnArithmeticDateScalar.txt | 98 +-
.../TimestampColumnArithmeticDateScalarBase.txt | 126 --
...pColumnArithmeticIntervalYearMonthColumn.txt | 59 +-
...pColumnArithmeticIntervalYearMonthScalar.txt | 41 +-
...TimestampColumnArithmeticTimestampColumn.txt | 128 ++-
...stampColumnArithmeticTimestampColumnBase.txt | 152 ---
...TimestampColumnArithmeticTimestampScalar.txt | 96 +-
...stampColumnArithmeticTimestampScalarBase.txt | 125 --
.../TimestampColumnCompareLongDoubleScalar.txt | 1 +
.../TimestampColumnCompareTimestampColumn.txt | 122 +-
...imestampColumnCompareTimestampColumnBase.txt | 140 ---
.../TimestampColumnCompareTimestampScalar.txt | 114 +-
...imestampColumnCompareTimestampScalarBase.txt | 131 ---
.../TimestampScalarArithmeticDateColumn.txt | 117 +-
.../TimestampScalarArithmeticDateColumnBase.txt | 151 ---
...pScalarArithmeticIntervalYearMonthColumn.txt | 62 +-
...TimestampScalarArithmeticTimestampColumn.txt | 103 +-
...stampScalarArithmeticTimestampColumnBase.txt | 139 ---
.../TimestampScalarCompareLongDoubleColumn.txt | 4 +-
.../TimestampScalarCompareTimestampColumn.txt | 115 +-
...imestampScalarCompareTimestampColumnBase.txt | 132 ---
.../VectorUDAFMinMaxIntervalDayTime.txt | 454 ++++++++
.../UDAFTemplates/VectorUDAFMinMaxTimestamp.txt | 31 +-
.../org/apache/hadoop/hive/llap/DebugUtils.java | 43 -
.../org/apache/hadoop/hive/llap/LogLevels.java | 53 -
.../java/org/apache/hadoop/hive/ql/Driver.java | 20 +
.../org/apache/hadoop/hive/ql/QueryDisplay.java | 11 +-
.../hadoop/hive/ql/exec/FunctionRegistry.java | 7 +
.../hadoop/hive/ql/exec/OperatorUtils.java | 45 +-
.../hive/ql/exec/OrcFileMergeOperator.java | 4 +-
.../persistence/HybridHashTableContainer.java | 40 +-
.../ql/exec/persistence/KeyValueContainer.java | 25 +-
.../ql/exec/persistence/ObjectContainer.java | 24 +-
.../hive/ql/exec/persistence/RowContainer.java | 34 +-
.../ql/exec/spark/HiveSparkClientFactory.java | 4 +
.../ql/exec/spark/SparkReduceRecordHandler.java | 2 +
.../hadoop/hive/ql/exec/tez/DagUtils.java | 1 +
.../hive/ql/exec/vector/TimestampUtils.java | 8 +
.../hive/ql/exec/vector/VectorAssignRow.java | 23 +-
.../exec/vector/VectorColumnAssignFactory.java | 19 +-
.../ql/exec/vector/VectorColumnSetInfo.java | 23 +-
.../hive/ql/exec/vector/VectorCopyRow.java | 32 +
.../ql/exec/vector/VectorDeserializeRow.java | 13 +-
.../exec/vector/VectorExpressionDescriptor.java | 6 +-
.../hive/ql/exec/vector/VectorExtractRow.java | 24 +-
.../ql/exec/vector/VectorGroupByOperator.java | 2 +-
.../ql/exec/vector/VectorGroupKeyHelper.java | 12 +
.../ql/exec/vector/VectorHashKeyWrapper.java | 82 +-
.../exec/vector/VectorHashKeyWrapperBatch.java | 112 +-
.../hive/ql/exec/vector/VectorSerializeRow.java | 7 +-
.../ql/exec/vector/VectorizationContext.java | 27 +-
.../ql/exec/vector/VectorizedBatchUtil.java | 20 +-
.../ql/exec/vector/VectorizedRowBatchCtx.java | 12 +-
.../expressions/CastDecimalToTimestamp.java | 3 +-
.../expressions/CastDoubleToTimestamp.java | 17 +-
.../vector/expressions/CastLongToTimestamp.java | 8 +-
.../CastMillisecondsLongToTimestamp.java | 22 +-
.../CastStringToIntervalDayTime.java | 8 +-
.../expressions/CastTimestampToBoolean.java | 4 +-
.../vector/expressions/CastTimestampToDate.java | 2 +-
.../expressions/CastTimestampToDecimal.java | 9 +-
.../expressions/CastTimestampToDouble.java | 13 +-
.../vector/expressions/CastTimestampToLong.java | 12 +-
.../expressions/ConstantVectorExpression.java | 36 +-
.../expressions/DateColSubtractDateColumn.java | 80 +-
.../expressions/DateColSubtractDateScalar.java | 51 +-
.../DateScalarSubtractDateColumn.java | 52 +-
.../FilterTimestampColumnInList.java | 27 +-
.../IfExprIntervalDayTimeColumnColumn.java | 103 +-
.../IfExprIntervalDayTimeColumnScalar.java | 94 +-
.../IfExprIntervalDayTimeScalarColumn.java | 96 +-
.../IfExprIntervalDayTimeScalarScalar.java | 84 +-
.../IfExprTimestampColumnColumnBase.java | 8 +-
.../IfExprTimestampColumnScalar.java | 3 +-
.../IfExprTimestampColumnScalarBase.java | 14 +-
.../IfExprTimestampScalarColumn.java | 3 +-
.../IfExprTimestampScalarColumnBase.java | 15 +-
.../IfExprTimestampScalarScalar.java | 3 +-
.../IfExprTimestampScalarScalarBase.java | 13 +-
.../ql/exec/vector/expressions/NullUtil.java | 26 +
.../expressions/TimestampColumnInList.java | 29 +-
.../expressions/VectorExpressionWriter.java | 6 +-
.../VectorExpressionWriterFactory.java | 124 +-
.../expressions/VectorUDFDateAddColCol.java | 2 +-
.../expressions/VectorUDFDateAddColScalar.java | 2 +-
.../expressions/VectorUDFDateDiffColCol.java | 10 +-
.../expressions/VectorUDFDateDiffColScalar.java | 2 +-
.../expressions/VectorUDFDateDiffScalarCol.java | 2 +-
.../expressions/VectorUDFDateTimestamp.java | 2 +-
.../expressions/VectorUDFUnixTimeStampDate.java | 7 +-
.../VectorUDFUnixTimeStampTimestamp.java | 5 +-
.../aggregates/VectorUDAFAvgTimestamp.java | 40 +-
.../aggregates/VectorUDAFStdPopTimestamp.java | 24 +-
.../aggregates/VectorUDAFStdSampTimestamp.java | 27 +-
.../aggregates/VectorUDAFVarPopTimestamp.java | 24 +-
.../aggregates/VectorUDAFVarSampTimestamp.java | 24 +-
.../mapjoin/VectorMapJoinCommonOperator.java | 2 +-
.../mapjoin/VectorMapJoinRowBytesContainer.java | 24 +-
.../ql/exec/vector/udf/VectorUDFAdaptor.java | 13 +-
.../hadoop/hive/ql/io/orc/RecordReaderImpl.java | 15 +-
.../hive/ql/io/orc/encoded/EncodedReader.java | 2 +-
.../ql/io/orc/encoded/EncodedReaderImpl.java | 131 +--
.../io/parquet/convert/HiveSchemaConverter.java | 10 +-
.../parquet/read/DataWritableReadSupport.java | 75 +-
.../ql/io/parquet/serde/ParquetHiveSerDe.java | 11 +-
.../apache/hadoop/hive/ql/metadata/Hive.java | 252 ++--
.../hadoop/hive/ql/optimizer/Optimizer.java | 4 +-
.../ql/optimizer/ReduceSinkMapJoinProc.java | 24 +-
.../hive/ql/optimizer/StatsOptimizer.java | 14 +-
.../rules/HivePointLookupOptimizerRule.java | 381 ++++++
.../ql/optimizer/pcr/PcrExprProcFactory.java | 103 +-
.../hive/ql/optimizer/physical/Vectorizer.java | 7 +
.../stats/annotation/StatsRulesProcFactory.java | 3 +-
.../hadoop/hive/ql/parse/CalcitePlanner.java | 40 +-
.../hive/ql/parse/DDLSemanticAnalyzer.java | 15 +-
.../hadoop/hive/ql/parse/GenTezUtils.java | 3 +-
.../hadoop/hive/ql/parse/ParseContext.java | 5 +
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 8 +-
.../hadoop/hive/ql/txn/compactor/Cleaner.java | 6 +
.../hive/ql/txn/compactor/CompactorThread.java | 5 +
.../hadoop/hive/ql/txn/compactor/Initiator.java | 9 +-
.../hadoop/hive/ql/txn/compactor/Worker.java | 9 +-
.../hadoop/hive/ql/util/DateTimeMath.java | 214 +++-
.../apache/hadoop/hive/ql/TestTxnCommands2.java | 47 +
.../ql/exec/persistence/TestHashPartition.java | 3 +-
.../TestTimestampWritableAndColumnVector.java | 68 ++
.../TestVectorExpressionWriters.java | 23 +-
.../TestVectorFilterExpressions.java | 1 -
.../expressions/TestVectorMathFunctions.java | 53 +-
.../TestVectorTimestampExpressions.java | 26 +-
.../vector/expressions/TestVectorTypeCasts.java | 216 ++--
.../TestVectorMapJoinRowBytesContainer.java | 3 +-
.../FakeVectorRowBatchFromObjectIterables.java | 3 +-
.../vector/util/VectorizedRowGroupGenUtil.java | 14 +-
.../hive/ql/io/orc/TestColumnStatistics.java | 5 +-
.../hive/ql/io/orc/TestInputOutputFormat.java | 2 +-
.../hadoop/hive/ql/io/orc/TestOrcFile.java | 5 +-
.../hive/ql/io/orc/TestVectorOrcFile.java | 22 +-
.../ql/optimizer/physical/TestVectorizer.java | 5 +
.../clientpositive/auto_join19_inclause.q | 18 +
.../clientpositive/bucket_map_join_tez1.q | 27 +
.../queries/clientpositive/filter_in_or_dup.q | 19 +
ql/src/test/queries/clientpositive/llap_udf.q | 6 +-
.../clientpositive/multi_insert_with_join.q | 29 +
.../test/queries/clientpositive/orc_merge12.q | 51 +
.../clientpositive/parquet_schema_evolution.q | 14 +
.../sample_islocalmode_hook_use_metadata.q | 48 +
.../clientpositive/vector_complex_join.q | 29 +
.../clientpositive/vector_interval_arithmetic.q | 174 +++
.../clientnegative/dbtxnmgr_nodblock.q.out | 2 +
.../clientnegative/dbtxnmgr_nodbunlock.q.out | 2 +
.../lockneg_query_tbl_in_locked_db.q.out | 6 +
.../lockneg_try_db_lock_conflict.q.out | 6 +
.../lockneg_try_drop_locked_db.q.out | 4 +
.../lockneg_try_lock_db_in_use.q.out | 6 +
.../clientpositive/auto_join19_inclause.q.out | 130 +++
.../clientpositive/constprog_semijoin.q.out | 4 +-
.../dynpart_sort_optimization_acid.q.out | 4 +-
.../clientpositive/filter_in_or_dup.q.out | 96 ++
.../llap/bucket_map_join_tez1.q.out | 308 +++++
.../clientpositive/multi_insert_with_join.q.out | 128 +++
.../results/clientpositive/orc_merge12.q.out | 606 ++++++++++
.../parquet_map_null.q.java1.8.out | 1 +
.../parquet_schema_evolution.q.out | 65 ++
.../clientpositive/parquet_type_promotion.q.out | 2 +-
.../results/clientpositive/perf/query13.q.out | 14 +-
.../results/clientpositive/perf/query27.q.out | 2 +-
.../results/clientpositive/perf/query34.q.out | 2 +-
.../results/clientpositive/perf/query48.q.out | 14 +-
.../results/clientpositive/perf/query68.q.out | 2 +-
.../results/clientpositive/perf/query73.q.out | 2 +-
.../results/clientpositive/perf/query79.q.out | 2 +-
.../results/clientpositive/perf/query82.q.out | 2 +-
.../results/clientpositive/perf/query85.q.out | 26 +-
.../results/clientpositive/pointlookup2.q.out | 38 +-
.../results/clientpositive/pointlookup3.q.out | 50 +-
.../results/clientpositive/pointlookup4.q.out | 2 +-
.../sample_islocalmode_hook_use_metadata.q.out | 230 ++++
.../spark/bucket_map_join_tez1.q.out | 306 +++++
.../spark/constprog_semijoin.q.out | 4 +-
.../spark/multi_insert_with_join.q.out | 128 +++
.../tez/bucket_map_join_tez1.q.out | 294 +++++
.../clientpositive/tez/bucketpruning1.q.out | 8 +-
.../clientpositive/tez/constprog_semijoin.q.out | 4 +-
.../clientpositive/tez/explainuser_1.q.out | 92 +-
.../clientpositive/tez/orc_merge12.q.out | 606 ++++++++++
.../tez/vector_complex_join.q.out | 227 ++++
.../tez/vector_interval_arithmetic.q.out | 1086 ++++++++++++++++++
.../tez/vector_mr_diff_schema_alias.q.out | 2 +-
.../clientpositive/tez/vectorized_casts.q.out | 18 +-
.../tez/vectorized_timestamp.q.out | 157 +++
.../clientpositive/vector_complex_join.q.out | 225 ++++
.../vector_interval_arithmetic.q.out | 1027 +++++++++++++++++
.../vector_mr_diff_schema_alias.q.out | 2 +-
.../clientpositive/vectorized_casts.q.out | 18 +-
.../hive/serde2/io/TimestampWritable.java | 71 +-
.../auth/LdapAuthenticationProviderImpl.java | 317 ++---
.../apache/hive/service/server/HiveServer2.java | 39 +-
.../auth/TestLdapAtnProviderWithMiniDS.java | 200 +++-
.../apache/hive/service/cli/CLIServiceTest.java | 8 +
.../org/apache/hadoop/fs/ProxyFileSystem.java | 5 +-
.../hive/common/type/HiveIntervalDayTime.java | 253 ++++
.../hadoop/hive/common/type/PisaTimestamp.java | 609 ----------
.../hadoop/hive/common/type/RandomTypeUtil.java | 70 +-
.../hive/ql/exec/vector/ColumnVector.java | 2 +-
.../vector/IntervalDayTimeColumnVector.java | 348 ++++++
.../ql/exec/vector/TimestampColumnVector.java | 341 ++----
.../hive/common/util/IntervalDayTimeUtils.java | 77 ++
.../hive/common/type/TestPisaTimestamp.java | 118 --
.../exec/vector/TestTimestampColumnVector.java | 117 ++
309 files changed, 16607 insertions(+), 7075 deletions(-)
----------------------------------------------------------------------
[38/50] [abbrv] hive git commit: revert HIVE-12531 : Implement
fast-path for Year/Month UDFs for dates between 1999 and 2038 (Jason Dere via
Sergey Shelukhin)
Posted by jd...@apache.org.
revert HIVE-12531 : Implement fast-path for Year/Month UDFs for dates between 1999 and 2038 (Jason Dere via Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/09b00fc8
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/09b00fc8
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/09b00fc8
Branch: refs/heads/llap
Commit: 09b00fc863d19cf513fe1d188bb671f370f64c2d
Parents: ff10f03
Author: Jason Dere <jd...@hortonworks.com>
Authored: Tue Mar 29 11:16:32 2016 -0700
Committer: Jason Dere <jd...@hortonworks.com>
Committed: Tue Mar 29 11:16:32 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/ql/udf/UDFMonth.java | 16 ++++++++--------
.../java/org/apache/hadoop/hive/ql/udf/UDFYear.java | 16 ++++++++--------
.../expressions/TestVectorDateExpressions.java | 13 +++----------
3 files changed, 19 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/09b00fc8/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFMonth.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFMonth.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFMonth.java
index 05afb8e..8c2b0e4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFMonth.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFMonth.java
@@ -20,8 +20,8 @@ package org.apache.hadoop.hive.ql.udf;
import java.text.ParseException;
import java.text.SimpleDateFormat;
+import java.util.Calendar;
import java.util.Date;
-import org.joda.time.MutableDateTime;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
@@ -53,7 +53,7 @@ import org.apache.hadoop.io.Text;
@NDV(maxNdv = 31)
public class UDFMonth extends UDF {
private final SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd");
- private transient final MutableDateTime mdt = new MutableDateTime();
+ private final Calendar calendar = Calendar.getInstance();
private final IntWritable result = new IntWritable();
@@ -75,8 +75,8 @@ public class UDFMonth extends UDF {
}
try {
Date date = formatter.parse(dateString.toString());
- mdt.setMillis(date.getTime());
- result.set(mdt.getMonthOfYear());
+ calendar.setTime(date);
+ result.set(1 + calendar.get(Calendar.MONTH));
return result;
} catch (ParseException e) {
return null;
@@ -88,8 +88,8 @@ public class UDFMonth extends UDF {
return null;
}
- mdt.setMillis(d.get().getTime());
- result.set(mdt.getMonthOfYear());
+ calendar.setTime(d.get());
+ result.set(1 + calendar.get(Calendar.MONTH));
return result;
}
@@ -98,8 +98,8 @@ public class UDFMonth extends UDF {
return null;
}
- mdt.setMillis(t.getTimestamp().getTime());
- result.set(mdt.getMonthOfYear());
+ calendar.setTime(t.getTimestamp());
+ result.set(1 + calendar.get(Calendar.MONTH));
return result;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/09b00fc8/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFYear.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFYear.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFYear.java
index fb3a655..d7ecd8c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFYear.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFYear.java
@@ -20,8 +20,8 @@ package org.apache.hadoop.hive.ql.udf;
import java.text.ParseException;
import java.text.SimpleDateFormat;
+import java.util.Calendar;
import java.util.Date;
-import org.joda.time.MutableDateTime;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
@@ -53,7 +53,7 @@ import org.apache.hadoop.io.Text;
@NDV(maxNdv = 20) // although technically its unbounded, its unlikely we will ever see ndv > 20
public class UDFYear extends UDF {
private final SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd");
- private transient final MutableDateTime mdt = new MutableDateTime();
+ private final Calendar calendar = Calendar.getInstance();
private final IntWritable result = new IntWritable();
@@ -77,8 +77,8 @@ public class UDFYear extends UDF {
try {
Date date = formatter.parse(dateString.toString());
- mdt.setMillis(date.getTime());
- result.set(mdt.getYear());
+ calendar.setTime(date);
+ result.set(calendar.get(Calendar.YEAR));
return result;
} catch (ParseException e) {
return null;
@@ -90,8 +90,8 @@ public class UDFYear extends UDF {
return null;
}
- mdt.setMillis(d.get().getTime());
- result.set(mdt.getYear());
+ calendar.setTime(d.get());
+ result.set(calendar.get(Calendar.YEAR));
return result;
}
@@ -100,8 +100,8 @@ public class UDFYear extends UDF {
return null;
}
- mdt.setMillis(t.getTimestamp().getTime());
- result.set(mdt.getYear());
+ calendar.setTime(t.getTimestamp());
+ result.set(calendar.get(Calendar.YEAR));
return result;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/09b00fc8/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorDateExpressions.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorDateExpressions.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorDateExpressions.java
index 61c96e9..58cecc1 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorDateExpressions.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorDateExpressions.java
@@ -38,9 +38,7 @@ import org.junit.Test;
import org.junit.internal.runners.statements.Fail;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import com.sun.tools.javac.resources.javac;
-import java.sql.Date;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Calendar;
@@ -55,7 +53,6 @@ import java.util.concurrent.ThreadFactory;
public class TestVectorDateExpressions {
private ExecutorService runner;
- private static final int MAX_SANE_DATE_VALUE = new DateWritable(Date.valueOf("3000-01-01")).getDays();
/* copied over from VectorUDFTimestampFieldLong */
private TimestampWritable toTimestampWritable(long daysSinceEpoch) {
@@ -81,15 +78,11 @@ public class TestVectorDateExpressions {
}
private VectorizedRowBatch getVectorizedRandomRowBatch(int seed, int size) {
- return getVectorizedRandomRowBatch(seed, size, Integer.MAX_VALUE);
- }
-
- private VectorizedRowBatch getVectorizedRandomRowBatch(int seed, int size, int maxValue) {
VectorizedRowBatch batch = new VectorizedRowBatch(2, size);
LongColumnVector lcv = new LongColumnVector(size);
Random rand = new Random(seed);
for (int i = 0; i < size; i++) {
- lcv.vector[i] = (rand.nextInt(maxValue));
+ lcv.vector[i] = (rand.nextInt());
}
batch.cols[0] = lcv;
batch.cols[1] = new LongColumnVector(size);
@@ -166,7 +159,7 @@ public class TestVectorDateExpressions {
batch.cols[0].isNull[0] = true;
verifyUDFYear(batch);
- batch = getVectorizedRandomRowBatch(200, VectorizedRowBatch.DEFAULT_SIZE, MAX_SANE_DATE_VALUE);
+ batch = getVectorizedRandomRowBatch(200, VectorizedRowBatch.DEFAULT_SIZE);
verifyUDFYear(batch);
TestVectorizedRowBatch.addRandomNulls(batch.cols[0]);
verifyUDFYear(batch);
@@ -290,7 +283,7 @@ public class TestVectorDateExpressions {
batch.cols[0].isNull[0] = true;
verifyUDFMonth(batch);
- batch = getVectorizedRandomRowBatch(200, VectorizedRowBatch.DEFAULT_SIZE, MAX_SANE_DATE_VALUE);
+ batch = getVectorizedRandomRowBatch(200, VectorizedRowBatch.DEFAULT_SIZE);
verifyUDFMonth(batch);
TestVectorizedRowBatch.addRandomNulls(batch.cols[0]);
verifyUDFMonth(batch);
[04/50] [abbrv] hive git commit: HIVE-13325: Excessive logging when
ORC PPD fails type conversions (Prasanth Jayachandran reviewed by Gopal V)
Posted by jd...@apache.org.
HIVE-13325: Excessive logging when ORC PPD fails type conversions (Prasanth Jayachandran reviewed by Gopal V)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d3a5f20b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d3a5f20b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d3a5f20b
Branch: refs/heads/llap
Commit: d3a5f20b4487e241b3e9424d1d762dfca0c25d2f
Parents: d469e61
Author: Prasanth Jayachandran <pr...@apache.org>
Authored: Thu Mar 24 13:30:55 2016 -0500
Committer: Prasanth Jayachandran <pr...@apache.org>
Committed: Thu Mar 24 13:31:08 2016 -0500
----------------------------------------------------------------------
.../hadoop/hive/ql/io/orc/RecordReaderImpl.java | 15 ++++++++++++---
1 file changed, 12 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/d3a5f20b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
index d511df6..aa835ae 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
@@ -378,9 +378,9 @@ public class RecordReaderImpl implements RecordReader {
}
TruthValue result;
+ Object baseObj = predicate.getLiteral();
try {
// Predicate object and stats objects are converted to the type of the predicate object.
- Object baseObj = predicate.getLiteral();
Object minValue = getBaseObjectForComparison(predicate.getType(), min);
Object maxValue = getBaseObjectForComparison(predicate.getType(), max);
Object predObj = getBaseObjectForComparison(predicate.getType(), baseObj);
@@ -392,8 +392,17 @@ public class RecordReaderImpl implements RecordReader {
// in case failed conversion, return the default YES_NO_NULL truth value
} catch (Exception e) {
if (LOG.isWarnEnabled()) {
- LOG.warn("Exception when evaluating predicate. Skipping ORC PPD." +
- " Exception: " + ExceptionUtils.getStackTrace(e));
+ final String statsType = min == null ?
+ (max == null ? "null" : max.getClass().getSimpleName()) :
+ min.getClass().getSimpleName();
+ final String predicateType = baseObj == null ? "null" : baseObj.getClass().getSimpleName();
+ final String reason = e.getClass().getSimpleName() + " when evaluating predicate." +
+ " Skipping ORC PPD." +
+ " Exception: " + e.getMessage() +
+ " StatsType: " + statsType +
+ " PredicateType: " + predicateType;
+ LOG.warn(reason);
+ LOG.debug(reason, e);
}
if (predicate.getOperator().equals(PredicateLeaf.Operator.NULL_SAFE_EQUALS) || !hasNull) {
result = TruthValue.YES_NO;
[07/50] [abbrv] hive git commit: HIVE-13362: Commit binary file
required for HIVE-13361 (Prasanth Jayachandran reviewed by Gopal V)
Posted by jd...@apache.org.
HIVE-13362: Commit binary file required for HIVE-13361 (Prasanth Jayachandran reviewed by Gopal V)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/dfba1fb2
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/dfba1fb2
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/dfba1fb2
Branch: refs/heads/llap
Commit: dfba1fb280f82822c1c006a0961a3ce9a52b6a6d
Parents: ab095f0
Author: Prasanth Jayachandran <pr...@apache.org>
Authored: Thu Mar 24 20:09:14 2016 -0500
Committer: Prasanth Jayachandran <pr...@apache.org>
Committed: Thu Mar 24 20:10:26 2016 -0500
----------------------------------------------------------------------
data/files/alltypesorc3xcols | Bin 0 -> 1504592 bytes
1 file changed, 0 insertions(+), 0 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/dfba1fb2/data/files/alltypesorc3xcols
----------------------------------------------------------------------
diff --git a/data/files/alltypesorc3xcols b/data/files/alltypesorc3xcols
new file mode 100644
index 0000000..e484873
Binary files /dev/null and b/data/files/alltypesorc3xcols differ
[14/50] [abbrv] hive git commit: HIVE-12531 : Implement fast-path for
Year/Month UDFs for dates between 1999 and 2038 (Jason Dere via Sergey
Shelukhin)
Posted by jd...@apache.org.
HIVE-12531 : Implement fast-path for Year/Month UDFs for dates between 1999 and 2038 (Jason Dere via Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e384b2b6
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e384b2b6
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e384b2b6
Branch: refs/heads/llap
Commit: e384b2b657c819d5963b8f76222f78bb479a29a2
Parents: b75d9ea
Author: Jason Dere <jd...@hortonworks.com>
Authored: Wed Dec 9 11:48:00 2015 -0800
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Fri Mar 25 07:21:55 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/ql/udf/UDFMonth.java | 16 ++++++++--------
.../java/org/apache/hadoop/hive/ql/udf/UDFYear.java | 16 ++++++++--------
.../expressions/TestVectorDateExpressions.java | 13 ++++++++++---
3 files changed, 26 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/e384b2b6/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFMonth.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFMonth.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFMonth.java
index 8c2b0e4..05afb8e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFMonth.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFMonth.java
@@ -20,8 +20,8 @@ package org.apache.hadoop.hive.ql.udf;
import java.text.ParseException;
import java.text.SimpleDateFormat;
-import java.util.Calendar;
import java.util.Date;
+import org.joda.time.MutableDateTime;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
@@ -53,7 +53,7 @@ import org.apache.hadoop.io.Text;
@NDV(maxNdv = 31)
public class UDFMonth extends UDF {
private final SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd");
- private final Calendar calendar = Calendar.getInstance();
+ private transient final MutableDateTime mdt = new MutableDateTime();
private final IntWritable result = new IntWritable();
@@ -75,8 +75,8 @@ public class UDFMonth extends UDF {
}
try {
Date date = formatter.parse(dateString.toString());
- calendar.setTime(date);
- result.set(1 + calendar.get(Calendar.MONTH));
+ mdt.setMillis(date.getTime());
+ result.set(mdt.getMonthOfYear());
return result;
} catch (ParseException e) {
return null;
@@ -88,8 +88,8 @@ public class UDFMonth extends UDF {
return null;
}
- calendar.setTime(d.get());
- result.set(1 + calendar.get(Calendar.MONTH));
+ mdt.setMillis(d.get().getTime());
+ result.set(mdt.getMonthOfYear());
return result;
}
@@ -98,8 +98,8 @@ public class UDFMonth extends UDF {
return null;
}
- calendar.setTime(t.getTimestamp());
- result.set(1 + calendar.get(Calendar.MONTH));
+ mdt.setMillis(t.getTimestamp().getTime());
+ result.set(mdt.getMonthOfYear());
return result;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/e384b2b6/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFYear.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFYear.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFYear.java
index d7ecd8c..fb3a655 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFYear.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFYear.java
@@ -20,8 +20,8 @@ package org.apache.hadoop.hive.ql.udf;
import java.text.ParseException;
import java.text.SimpleDateFormat;
-import java.util.Calendar;
import java.util.Date;
+import org.joda.time.MutableDateTime;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
@@ -53,7 +53,7 @@ import org.apache.hadoop.io.Text;
@NDV(maxNdv = 20) // although technically its unbounded, its unlikely we will ever see ndv > 20
public class UDFYear extends UDF {
private final SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd");
- private final Calendar calendar = Calendar.getInstance();
+ private transient final MutableDateTime mdt = new MutableDateTime();
private final IntWritable result = new IntWritable();
@@ -77,8 +77,8 @@ public class UDFYear extends UDF {
try {
Date date = formatter.parse(dateString.toString());
- calendar.setTime(date);
- result.set(calendar.get(Calendar.YEAR));
+ mdt.setMillis(date.getTime());
+ result.set(mdt.getYear());
return result;
} catch (ParseException e) {
return null;
@@ -90,8 +90,8 @@ public class UDFYear extends UDF {
return null;
}
- calendar.setTime(d.get());
- result.set(calendar.get(Calendar.YEAR));
+ mdt.setMillis(d.get().getTime());
+ result.set(mdt.getYear());
return result;
}
@@ -100,8 +100,8 @@ public class UDFYear extends UDF {
return null;
}
- calendar.setTime(t.getTimestamp());
- result.set(calendar.get(Calendar.YEAR));
+ mdt.setMillis(t.getTimestamp().getTime());
+ result.set(mdt.getYear());
return result;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/e384b2b6/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorDateExpressions.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorDateExpressions.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorDateExpressions.java
index 58cecc1..61c96e9 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorDateExpressions.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorDateExpressions.java
@@ -38,7 +38,9 @@ import org.junit.Test;
import org.junit.internal.runners.statements.Fail;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.sun.tools.javac.resources.javac;
+import java.sql.Date;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Calendar;
@@ -53,6 +55,7 @@ import java.util.concurrent.ThreadFactory;
public class TestVectorDateExpressions {
private ExecutorService runner;
+ private static final int MAX_SANE_DATE_VALUE = new DateWritable(Date.valueOf("3000-01-01")).getDays();
/* copied over from VectorUDFTimestampFieldLong */
private TimestampWritable toTimestampWritable(long daysSinceEpoch) {
@@ -78,11 +81,15 @@ public class TestVectorDateExpressions {
}
private VectorizedRowBatch getVectorizedRandomRowBatch(int seed, int size) {
+ return getVectorizedRandomRowBatch(seed, size, Integer.MAX_VALUE);
+ }
+
+ private VectorizedRowBatch getVectorizedRandomRowBatch(int seed, int size, int maxValue) {
VectorizedRowBatch batch = new VectorizedRowBatch(2, size);
LongColumnVector lcv = new LongColumnVector(size);
Random rand = new Random(seed);
for (int i = 0; i < size; i++) {
- lcv.vector[i] = (rand.nextInt());
+ lcv.vector[i] = (rand.nextInt(maxValue));
}
batch.cols[0] = lcv;
batch.cols[1] = new LongColumnVector(size);
@@ -159,7 +166,7 @@ public class TestVectorDateExpressions {
batch.cols[0].isNull[0] = true;
verifyUDFYear(batch);
- batch = getVectorizedRandomRowBatch(200, VectorizedRowBatch.DEFAULT_SIZE);
+ batch = getVectorizedRandomRowBatch(200, VectorizedRowBatch.DEFAULT_SIZE, MAX_SANE_DATE_VALUE);
verifyUDFYear(batch);
TestVectorizedRowBatch.addRandomNulls(batch.cols[0]);
verifyUDFYear(batch);
@@ -283,7 +290,7 @@ public class TestVectorDateExpressions {
batch.cols[0].isNull[0] = true;
verifyUDFMonth(batch);
- batch = getVectorizedRandomRowBatch(200, VectorizedRowBatch.DEFAULT_SIZE);
+ batch = getVectorizedRandomRowBatch(200, VectorizedRowBatch.DEFAULT_SIZE, MAX_SANE_DATE_VALUE);
verifyUDFMonth(batch);
TestVectorizedRowBatch.addRandomNulls(batch.cols[0]);
verifyUDFMonth(batch);
[06/50] [abbrv] hive git commit: HIVE-13008 - WebHcat DDL commands in
secure mode NPE when default FileSystem doesn't support delegation tokens
(Eugene Koifman, reviewed by Chris Nauroth, Thejas Nair)
Posted by jd...@apache.org.
HIVE-13008 - WebHcat DDL commands in secure mode NPE when default FileSystem doesn't support delegation tokens (Eugene Koifman, reviewed by Chris Nauroth, Thejas Nair)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ab095f0b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ab095f0b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ab095f0b
Branch: refs/heads/llap
Commit: ab095f0bc24447ab73843a1ae23a32f7b6c4bd1a
Parents: f9d1b6a
Author: Eugene Koifman <ek...@hortonworks.com>
Authored: Thu Mar 24 18:03:32 2016 -0700
Committer: Eugene Koifman <ek...@hortonworks.com>
Committed: Thu Mar 24 18:03:32 2016 -0700
----------------------------------------------------------------------
.../hcatalog/templeton/SecureProxySupport.java | 46 ++++++++++++++------
1 file changed, 33 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/ab095f0b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java
----------------------------------------------------------------------
diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java
index 2ac62c0..13f3c9b 100644
--- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java
+++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java
@@ -20,10 +20,14 @@ package org.apache.hive.hcatalog.templeton;
import java.io.File;
import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.security.PrivilegedExceptionAction;
+import java.util.Collection;
import java.util.List;
import java.util.Map;
+import org.apache.commons.lang3.ArrayUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@@ -79,7 +83,7 @@ public class SecureProxySupport {
this.user = user;
File t = File.createTempFile("templeton", null);
tokenPath = new Path(t.toURI());
- Token fsToken = getFSDelegationToken(user, conf);
+ Token[] fsToken = getFSDelegationToken(user, conf);
String hcatTokenStr;
try {
hcatTokenStr = buildHcatDelegationToken(user);
@@ -130,11 +134,11 @@ public class SecureProxySupport {
}
}
- class TokenWrapper {
- Token<?> token;
+ private static class TokenWrapper {
+ Token<?>[] tokens = new Token<?>[0];
}
- private Token<?> getFSDelegationToken(String user,
+ private Token<?>[] getFSDelegationToken(String user,
final Configuration conf)
throws IOException, InterruptedException {
LOG.info("user: " + user + " loginUser: " + UserGroupInformation.getLoginUser().getUserName());
@@ -142,18 +146,32 @@ public class SecureProxySupport {
final TokenWrapper twrapper = new TokenWrapper();
ugi.doAs(new PrivilegedExceptionAction<Object>() {
- public Object run() throws IOException {
- FileSystem fs = FileSystem.get(conf);
- //todo: according to JavaDoc this seems like private API: addDelegationToken should be used
- twrapper.token = fs.getDelegationToken(ugi.getShortUserName());
+ public Object run() throws IOException, URISyntaxException {
+ Credentials creds = new Credentials();
+ //get Tokens for default FS. Not all FSs support delegation tokens, e.g. WASB
+ collectTokens(FileSystem.get(conf), twrapper, creds, ugi.getShortUserName());
+ //get tokens for all other known FSs since Hive tables may result in different ones
+ //passing "creds" prevents duplicate tokens from being added
+ Collection<String> URIs = conf.getStringCollection("mapreduce.job.hdfs-servers");
+ for(String uri : URIs) {
+ LOG.debug("Getting tokens for " + uri);
+ collectTokens(FileSystem.get(new URI(uri), conf), twrapper, creds, ugi.getShortUserName());
+ }
return null;
}
});
- return twrapper.token;
-
+ return twrapper.tokens;
}
-
- private void writeProxyDelegationTokens(final Token<?> fsToken,
+ private static void collectTokens(FileSystem fs, TokenWrapper twrapper, Credentials creds, String userName) throws IOException {
+ Token[] tokens = fs.addDelegationTokens(userName, creds);
+ if(tokens != null && tokens.length > 0) {
+ twrapper.tokens = ArrayUtils.addAll(twrapper.tokens, tokens);
+ }
+ }
+ /**
+ * @param fsTokens not null
+ */
+ private void writeProxyDelegationTokens(final Token<?> fsTokens[],
final Token<?> msToken,
final Configuration conf,
String user,
@@ -168,7 +186,9 @@ public class SecureProxySupport {
ugi.doAs(new PrivilegedExceptionAction<Object>() {
public Object run() throws IOException {
Credentials cred = new Credentials();
- cred.addToken(fsToken.getService(), fsToken);
+ for(Token<?> fsToken : fsTokens) {
+ cred.addToken(fsToken.getService(), fsToken);
+ }
cred.addToken(msToken.getService(), msToken);
cred.writeTokenStorageFile(tokenPath, conf);
return null;
[03/50] [abbrv] hive git commit: HIVE-12616 : NullPointerException
when spark session is reused to run a mapjoin (Nemon Lou, via Szehon)
Posted by jd...@apache.org.
HIVE-12616 : NullPointerException when spark session is reused to run a mapjoin (Nemon Lou, via Szehon)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d469e611
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d469e611
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d469e611
Branch: refs/heads/llap
Commit: d469e61108a1844fcc173674bfb2cd9f7ad01c18
Parents: 219d352
Author: Szehon Ho <sz...@cloudera.com>
Authored: Thu Mar 24 11:12:08 2016 -0700
Committer: Szehon Ho <sz...@cloudera.com>
Committed: Thu Mar 24 11:12:50 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/d469e611/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java
index 1798622..2427321 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java
@@ -28,6 +28,7 @@ import java.util.Set;
import org.apache.commons.compress.utils.CharsetNames;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.hive.ql.session.SessionState;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.hbase.HBaseConfiguration;
@@ -120,6 +121,9 @@ public class HiveSparkClientFactory {
sparkMaster = sparkConf.get("spark.master");
hiveConf.set("spark.master", sparkMaster);
}
+ if (SessionState.get() != null && SessionState.get().getConf() != null) {
+ SessionState.get().getConf().set("spark.master", sparkMaster);
+ }
if (sparkMaster.equals("yarn-cluster")) {
sparkConf.put("spark.yarn.maxAppAttempts", "1");
}
[45/50] [abbrv] hive git commit: HIVE-13361: Orc concatenation should
enforce the compression buffer size (Prasanth Jayachandran reviewed by Gopal
V)
Posted by jd...@apache.org.
HIVE-13361: Orc concatenation should enforce the compression buffer size (Prasanth Jayachandran reviewed by Gopal V)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8c1f055d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8c1f055d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8c1f055d
Branch: refs/heads/llap
Commit: 8c1f055d91d5dd50544e7d39f2dc1ad087ac78e2
Parents: b431c27
Author: Prasanth Jayachandran <pr...@apache.org>
Authored: Tue Mar 29 22:20:20 2016 -0700
Committer: Prasanth Jayachandran <pr...@apache.org>
Committed: Tue Mar 29 22:20:20 2016 -0700
----------------------------------------------------------------------
.../test/resources/testconfiguration.properties | 1 +
orc/src/java/org/apache/orc/OrcFile.java | 21 +-
.../java/org/apache/orc/impl/WriterImpl.java | 18 +-
.../hive/ql/exec/OrcFileMergeOperator.java | 4 +-
.../test/queries/clientpositive/orc_merge12.q | 51 ++
.../results/clientpositive/orc_merge12.q.out | 606 +++++++++++++++++++
.../clientpositive/tez/orc_merge12.q.out | 606 +++++++++++++++++++
7 files changed, 1295 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/8c1f055d/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index ed26dea..232f84e 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -175,6 +175,7 @@ minitez.query.files.shared=acid_globallimit.q,\
orc_merge9.q,\
orc_merge10.q,\
orc_merge11.q,\
+ orc_merge12.q,\
orc_merge_incompat1.q,\
orc_merge_incompat2.q,\
orc_merge_incompat3.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/8c1f055d/orc/src/java/org/apache/orc/OrcFile.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/OrcFile.java b/orc/src/java/org/apache/orc/OrcFile.java
index 3945a5d..85506ff 100644
--- a/orc/src/java/org/apache/orc/OrcFile.java
+++ b/orc/src/java/org/apache/orc/OrcFile.java
@@ -232,6 +232,7 @@ public class OrcFile {
private long blockSizeValue;
private int rowIndexStrideValue;
private int bufferSizeValue;
+ private boolean enforceBufferSize = false;
private boolean blockPaddingValue;
private CompressionKind compressValue;
private MemoryManager memoryManagerValue;
@@ -317,7 +318,10 @@ public class OrcFile {
/**
* The size of the memory buffers used for compressing and storing the
- * stripe in memory.
+ * stripe in memory. NOTE: ORC writer may choose to use smaller buffer
+ * size based on stripe size and number of columns for efficient stripe
+ * writing and memory utilization. To enforce writer to use the requested
+ * buffer size use enforceBufferSize().
*/
public WriterOptions bufferSize(int value) {
bufferSizeValue = value;
@@ -325,6 +329,17 @@ public class OrcFile {
}
/**
+ * Enforce writer to use requested buffer size instead of estimating
+ * buffer size based on stripe size and number of columns.
+ * See bufferSize() method for more info.
+ * Default: false
+ */
+ public WriterOptions enforceBufferSize() {
+ enforceBufferSize = true;
+ return this;
+ }
+
+ /**
* Sets whether the HDFS blocks are padded to prevent stripes from
* straddling blocks. Padding improves locality and thus the speed of
* reading, but costs space.
@@ -460,6 +475,10 @@ public class OrcFile {
return bufferSizeValue;
}
+ public boolean isEnforceBufferSize() {
+ return enforceBufferSize;
+ }
+
public int getRowIndexStride() {
return rowIndexStrideValue;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/8c1f055d/orc/src/java/org/apache/orc/impl/WriterImpl.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/impl/WriterImpl.java b/orc/src/java/org/apache/orc/impl/WriterImpl.java
index d4b9a14..f8afe06 100644
--- a/orc/src/java/org/apache/orc/impl/WriterImpl.java
+++ b/orc/src/java/org/apache/orc/impl/WriterImpl.java
@@ -180,8 +180,12 @@ public class WriterImpl implements Writer, MemoryManager.Callback {
buildIndex = rowIndexStride > 0;
codec = createCodec(compress);
int numColumns = schema.getMaximumId() + 1;
- this.bufferSize = getEstimatedBufferSize(defaultStripeSize,
- numColumns, opts.getBufferSize());
+ if (opts.isEnforceBufferSize()) {
+ this.bufferSize = opts.getBufferSize();
+ } else {
+ this.bufferSize = getEstimatedBufferSize(defaultStripeSize,
+ numColumns, opts.getBufferSize());
+ }
if (version == OrcFile.Version.V_0_11) {
/* do not write bloom filters for ORC v11 */
this.bloomFilterColumns = new boolean[schema.getMaximumId() + 1];
@@ -199,7 +203,7 @@ public class WriterImpl implements Writer, MemoryManager.Callback {
// ensure that we are able to handle callbacks before we register ourselves
memoryManager.addWriter(path, opts.getStripeSize(), this);
LOG.info("ORC writer created for path: {} with stripeSize: {} blockSize: {}" +
- " compression: {} estimatedBufferSize: {}", path, defaultStripeSize, blockSize,
+ " compression: {} bufferSize: {}", path, defaultStripeSize, blockSize,
compress, bufferSize);
}
@@ -212,13 +216,7 @@ public class WriterImpl implements Writer, MemoryManager.Callback {
// sizes.
int estBufferSize = (int) (stripeSize / (20 * numColumns));
estBufferSize = getClosestBufferSize(estBufferSize);
- if (estBufferSize > bs) {
- estBufferSize = bs;
- } else {
- LOG.info("WIDE TABLE - Number of columns: " + numColumns +
- " Chosen compression buffer size: " + estBufferSize);
- }
- return estBufferSize;
+ return estBufferSize > bs ? bs : estBufferSize;
}
private static int getClosestBufferSize(int estBufferSize) {
http://git-wip-us.apache.org/repos/asf/hive/blob/8c1f055d/ql/src/java/org/apache/hadoop/hive/ql/exec/OrcFileMergeOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/OrcFileMergeOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/OrcFileMergeOperator.java
index e554ab1..e3cb765 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/OrcFileMergeOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/OrcFileMergeOperator.java
@@ -121,7 +121,9 @@ public class OrcFileMergeOperator extends
.inspector(reader.getObjectInspector());
// compression buffer size should only be set if compression is enabled
if (compression != CompressionKind.NONE) {
- options.bufferSize(compressBuffSize);
+ // enforce is required to retain the buffer sizes of old files instead of orc writer
+ // inferring the optimal buffer size
+ options.bufferSize(compressBuffSize).enforceBufferSize();
}
outWriter = OrcFile.createWriter(outPath, options);
http://git-wip-us.apache.org/repos/asf/hive/blob/8c1f055d/ql/src/test/queries/clientpositive/orc_merge12.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/orc_merge12.q b/ql/src/test/queries/clientpositive/orc_merge12.q
new file mode 100644
index 0000000..ed17cea
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/orc_merge12.q
@@ -0,0 +1,51 @@
+CREATE TABLE `alltypesorc3xcols`(
+ `atinyint` tinyint,
+ `asmallint` smallint,
+ `aint` int,
+ `abigint` bigint,
+ `afloat` float,
+ `adouble` double,
+ `astring1` string,
+ `astring2` string,
+ `atimestamp1` timestamp,
+ `atimestamp2` timestamp,
+ `aboolean1` boolean,
+ `aboolean2` boolean,
+ `btinyint` tinyint,
+ `bsmallint` smallint,
+ `bint` int,
+ `bbigint` bigint,
+ `bfloat` float,
+ `bdouble` double,
+ `bstring1` string,
+ `bstring2` string,
+ `btimestamp1` timestamp,
+ `btimestamp2` timestamp,
+ `bboolean1` boolean,
+ `bboolean2` boolean,
+ `ctinyint` tinyint,
+ `csmallint` smallint,
+ `cint` int,
+ `cbigint` bigint,
+ `cfloat` float,
+ `cdouble` double,
+ `cstring1` string,
+ `cstring2` string,
+ `ctimestamp1` timestamp,
+ `ctimestamp2` timestamp,
+ `cboolean1` boolean,
+ `cboolean2` boolean) stored as ORC;
+
+load data local inpath '../../data/files/alltypesorc3xcols' into table alltypesorc3xcols;
+load data local inpath '../../data/files/alltypesorc3xcols' into table alltypesorc3xcols;
+
+select count(*) from alltypesorc3xcols;
+select sum(hash(*)) from alltypesorc3xcols;
+
+alter table alltypesorc3xcols concatenate;
+
+select count(*) from alltypesorc3xcols;
+select sum(hash(*)) from alltypesorc3xcols;
+
+SET hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.PostExecOrcFileDump;
+select * from alltypesorc3xcols limit 1;
http://git-wip-us.apache.org/repos/asf/hive/blob/8c1f055d/ql/src/test/results/clientpositive/orc_merge12.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/orc_merge12.q.out b/ql/src/test/results/clientpositive/orc_merge12.q.out
new file mode 100644
index 0000000..f23be5a
--- /dev/null
+++ b/ql/src/test/results/clientpositive/orc_merge12.q.out
@@ -0,0 +1,606 @@
+PREHOOK: query: CREATE TABLE `alltypesorc3xcols`(
+ `atinyint` tinyint,
+ `asmallint` smallint,
+ `aint` int,
+ `abigint` bigint,
+ `afloat` float,
+ `adouble` double,
+ `astring1` string,
+ `astring2` string,
+ `atimestamp1` timestamp,
+ `atimestamp2` timestamp,
+ `aboolean1` boolean,
+ `aboolean2` boolean,
+ `btinyint` tinyint,
+ `bsmallint` smallint,
+ `bint` int,
+ `bbigint` bigint,
+ `bfloat` float,
+ `bdouble` double,
+ `bstring1` string,
+ `bstring2` string,
+ `btimestamp1` timestamp,
+ `btimestamp2` timestamp,
+ `bboolean1` boolean,
+ `bboolean2` boolean,
+ `ctinyint` tinyint,
+ `csmallint` smallint,
+ `cint` int,
+ `cbigint` bigint,
+ `cfloat` float,
+ `cdouble` double,
+ `cstring1` string,
+ `cstring2` string,
+ `ctimestamp1` timestamp,
+ `ctimestamp2` timestamp,
+ `cboolean1` boolean,
+ `cboolean2` boolean) stored as ORC
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@alltypesorc3xcols
+POSTHOOK: query: CREATE TABLE `alltypesorc3xcols`(
+ `atinyint` tinyint,
+ `asmallint` smallint,
+ `aint` int,
+ `abigint` bigint,
+ `afloat` float,
+ `adouble` double,
+ `astring1` string,
+ `astring2` string,
+ `atimestamp1` timestamp,
+ `atimestamp2` timestamp,
+ `aboolean1` boolean,
+ `aboolean2` boolean,
+ `btinyint` tinyint,
+ `bsmallint` smallint,
+ `bint` int,
+ `bbigint` bigint,
+ `bfloat` float,
+ `bdouble` double,
+ `bstring1` string,
+ `bstring2` string,
+ `btimestamp1` timestamp,
+ `btimestamp2` timestamp,
+ `bboolean1` boolean,
+ `bboolean2` boolean,
+ `ctinyint` tinyint,
+ `csmallint` smallint,
+ `cint` int,
+ `cbigint` bigint,
+ `cfloat` float,
+ `cdouble` double,
+ `cstring1` string,
+ `cstring2` string,
+ `ctimestamp1` timestamp,
+ `ctimestamp2` timestamp,
+ `cboolean1` boolean,
+ `cboolean2` boolean) stored as ORC
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@alltypesorc3xcols
+PREHOOK: query: load data local inpath '../../data/files/alltypesorc3xcols' into table alltypesorc3xcols
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@alltypesorc3xcols
+POSTHOOK: query: load data local inpath '../../data/files/alltypesorc3xcols' into table alltypesorc3xcols
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@alltypesorc3xcols
+PREHOOK: query: load data local inpath '../../data/files/alltypesorc3xcols' into table alltypesorc3xcols
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@alltypesorc3xcols
+POSTHOOK: query: load data local inpath '../../data/files/alltypesorc3xcols' into table alltypesorc3xcols
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@alltypesorc3xcols
+PREHOOK: query: select count(*) from alltypesorc3xcols
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc3xcols
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from alltypesorc3xcols
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc3xcols
+#### A masked pattern was here ####
+24576
+PREHOOK: query: select sum(hash(*)) from alltypesorc3xcols
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc3xcols
+#### A masked pattern was here ####
+POSTHOOK: query: select sum(hash(*)) from alltypesorc3xcols
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc3xcols
+#### A masked pattern was here ####
+-302946892512
+PREHOOK: query: alter table alltypesorc3xcols concatenate
+PREHOOK: type: ALTER_TABLE_MERGE
+PREHOOK: Input: default@alltypesorc3xcols
+PREHOOK: Output: default@alltypesorc3xcols
+POSTHOOK: query: alter table alltypesorc3xcols concatenate
+POSTHOOK: type: ALTER_TABLE_MERGE
+POSTHOOK: Input: default@alltypesorc3xcols
+POSTHOOK: Output: default@alltypesorc3xcols
+PREHOOK: query: select count(*) from alltypesorc3xcols
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc3xcols
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from alltypesorc3xcols
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc3xcols
+#### A masked pattern was here ####
+24576
+PREHOOK: query: select sum(hash(*)) from alltypesorc3xcols
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc3xcols
+#### A masked pattern was here ####
+POSTHOOK: query: select sum(hash(*)) from alltypesorc3xcols
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc3xcols
+#### A masked pattern was here ####
+-302946892512
+PREHOOK: query: select * from alltypesorc3xcols limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc3xcols
+#### A masked pattern was here ####
+-- BEGIN ORC FILE DUMP --
+#### A masked pattern was here ####
+File Version: 0.12 with HIVE_13083
+Rows: 24576
+Compression: ZLIB
+Compression size: 262144
+Type: struct<_col0:tinyint,_col1:smallint,_col2:int,_col3:bigint,_col4:float,_col5:double,_col6:string,_col7:string,_col8:timestamp,_col9:timestamp,_col10:boolean,_col11:boolean,_col12:tinyint,_col13:smallint,_col14:int,_col15:bigint,_col16:float,_col17:double,_col18:string,_col19:string,_col20:timestamp,_col21:timestamp,_col22:boolean,_col23:boolean,_col24:tinyint,_col25:smallint,_col26:int,_col27:bigint,_col28:float,_col29:double,_col30:string,_col31:string,_col32:timestamp,_col33:timestamp,_col34:boolean,_col35:boolean>
+
+Stripe Statistics:
+ Stripe 1:
+ Column 0: count: 12288 hasNull: false
+ Column 1: count: 9173 hasNull: true min: -64 max: 62 sum: -39856
+ Column 2: count: 9174 hasNull: true min: -16379 max: 16376 sum: 7435990
+ Column 3: count: 9173 hasNull: true min: -1073279343 max: 1073680599 sum: 1438050863785
+ Column 4: count: 9173 hasNull: true min: -2147311592 max: 2145498388 sum: -1698460028409
+ Column 5: count: 9173 hasNull: true min: -64.0 max: 79.5530014038086 sum: -39479.635992884636
+ Column 6: count: 9174 hasNull: true min: -16379.0 max: 9763215.5639 sum: 5.62236530305E7
+ Column 7: count: 12288 hasNull: false min: 00020767-dd8f-4f4d-bd68-4b7be64b8e44 max: fffa3516-e219-4027-b0d3-72bb2e676c52 sum: 442368
+ Column 8: count: 12288 hasNull: false min: 000976f7-7075-4f3f-a564-5a375fafcc101416a2b7-7f64-41b7-851f-97d15405037e max: fffd0642-5f01-48cd-8d97-3428faee49e9b39f2b4c-efdc-4e5f-9ab5-4aa5394cb156 sum: 884736
+ Column 9: count: 9173 hasNull: true min: 1969-12-31 13:59:30.929 max: 1969-12-31 14:00:30.808
+ Column 10: count: 9174 hasNull: true min: 1969-12-31 13:59:30.929 max: 1969-12-31 14:00:30.808
+ Column 11: count: 9174 hasNull: true true: 6138
+ Column 12: count: 9173 hasNull: true true: 3983
+ Column 13: count: 9173 hasNull: true min: -64 max: 62 sum: -39856
+ Column 14: count: 9174 hasNull: true min: -16379 max: 16376 sum: 7435990
+ Column 15: count: 9173 hasNull: true min: -1073279343 max: 1073680599 sum: 1438050863785
+ Column 16: count: 9173 hasNull: true min: -2147311592 max: 2145498388 sum: -1698460028409
+ Column 17: count: 9173 hasNull: true min: -64.0 max: 79.5530014038086 sum: -39479.635992884636
+ Column 18: count: 9174 hasNull: true min: -16379.0 max: 9763215.5639 sum: 5.62236530305E7
+ Column 19: count: 9174 hasNull: true min: 0042l0d5rPD6sMlJ7Ue0q max: yy2GiGM sum: 127881
+ Column 20: count: 9173 hasNull: true min: 0034fkcXMQI3 max: yyt0S8WorA sum: 149134
+ Column 21: count: 9173 hasNull: true min: 1969-12-31 13:59:30.929 max: 1969-12-31 14:00:30.808
+ Column 22: count: 9174 hasNull: true min: 1969-12-31 13:59:30.929 max: 1969-12-31 14:00:30.808
+ Column 23: count: 9174 hasNull: true true: 6138
+ Column 24: count: 9173 hasNull: true true: 3983
+ Column 25: count: 9173 hasNull: true min: -64 max: 62 sum: -39856
+ Column 26: count: 9174 hasNull: true min: -16379 max: 16376 sum: 7435990
+ Column 27: count: 9173 hasNull: true min: -1073279343 max: 1073680599 sum: 1438050863785
+ Column 28: count: 9173 hasNull: true min: -2147311592 max: 2145498388 sum: -1698460028409
+ Column 29: count: 9173 hasNull: true min: -64.0 max: 79.5530014038086 sum: -39479.635992884636
+ Column 30: count: 9174 hasNull: true min: -16379.0 max: 9763215.5639 sum: 5.62236530305E7
+ Column 31: count: 9174 hasNull: true min: 0042l0d5rPD6sMlJ7Ue0q max: yy2GiGM sum: 127881
+ Column 32: count: 9173 hasNull: true min: 0034fkcXMQI3 max: yyt0S8WorA sum: 149134
+ Column 33: count: 9173 hasNull: true min: 1969-12-31 13:59:30.929 max: 1969-12-31 14:00:30.808
+ Column 34: count: 9174 hasNull: true min: 1969-12-31 13:59:30.929 max: 1969-12-31 14:00:30.808
+ Column 35: count: 9174 hasNull: true true: 6138
+ Column 36: count: 9173 hasNull: true true: 3983
+ Stripe 2:
+ Column 0: count: 12288 hasNull: false
+ Column 1: count: 9173 hasNull: true min: -64 max: 62 sum: -39856
+ Column 2: count: 9174 hasNull: true min: -16379 max: 16376 sum: 7435990
+ Column 3: count: 9173 hasNull: true min: -1073279343 max: 1073680599 sum: 1438050863785
+ Column 4: count: 9173 hasNull: true min: -2147311592 max: 2145498388 sum: -1698460028409
+ Column 5: count: 9173 hasNull: true min: -64.0 max: 79.5530014038086 sum: -39479.635992884636
+ Column 6: count: 9174 hasNull: true min: -16379.0 max: 9763215.5639 sum: 5.62236530305E7
+ Column 7: count: 12288 hasNull: false min: 00020767-dd8f-4f4d-bd68-4b7be64b8e44 max: fffa3516-e219-4027-b0d3-72bb2e676c52 sum: 442368
+ Column 8: count: 12288 hasNull: false min: 000976f7-7075-4f3f-a564-5a375fafcc101416a2b7-7f64-41b7-851f-97d15405037e max: fffd0642-5f01-48cd-8d97-3428faee49e9b39f2b4c-efdc-4e5f-9ab5-4aa5394cb156 sum: 884736
+ Column 9: count: 9173 hasNull: true min: 1969-12-31 13:59:30.929 max: 1969-12-31 14:00:30.808
+ Column 10: count: 9174 hasNull: true min: 1969-12-31 13:59:30.929 max: 1969-12-31 14:00:30.808
+ Column 11: count: 9174 hasNull: true true: 6138
+ Column 12: count: 9173 hasNull: true true: 3983
+ Column 13: count: 9173 hasNull: true min: -64 max: 62 sum: -39856
+ Column 14: count: 9174 hasNull: true min: -16379 max: 16376 sum: 7435990
+ Column 15: count: 9173 hasNull: true min: -1073279343 max: 1073680599 sum: 1438050863785
+ Column 16: count: 9173 hasNull: true min: -2147311592 max: 2145498388 sum: -1698460028409
+ Column 17: count: 9173 hasNull: true min: -64.0 max: 79.5530014038086 sum: -39479.635992884636
+ Column 18: count: 9174 hasNull: true min: -16379.0 max: 9763215.5639 sum: 5.62236530305E7
+ Column 19: count: 9174 hasNull: true min: 0042l0d5rPD6sMlJ7Ue0q max: yy2GiGM sum: 127881
+ Column 20: count: 9173 hasNull: true min: 0034fkcXMQI3 max: yyt0S8WorA sum: 149134
+ Column 21: count: 9173 hasNull: true min: 1969-12-31 13:59:30.929 max: 1969-12-31 14:00:30.808
+ Column 22: count: 9174 hasNull: true min: 1969-12-31 13:59:30.929 max: 1969-12-31 14:00:30.808
+ Column 23: count: 9174 hasNull: true true: 6138
+ Column 24: count: 9173 hasNull: true true: 3983
+ Column 25: count: 9173 hasNull: true min: -64 max: 62 sum: -39856
+ Column 26: count: 9174 hasNull: true min: -16379 max: 16376 sum: 7435990
+ Column 27: count: 9173 hasNull: true min: -1073279343 max: 1073680599 sum: 1438050863785
+ Column 28: count: 9173 hasNull: true min: -2147311592 max: 2145498388 sum: -1698460028409
+ Column 29: count: 9173 hasNull: true min: -64.0 max: 79.5530014038086 sum: -39479.635992884636
+ Column 30: count: 9174 hasNull: true min: -16379.0 max: 9763215.5639 sum: 5.62236530305E7
+ Column 31: count: 9174 hasNull: true min: 0042l0d5rPD6sMlJ7Ue0q max: yy2GiGM sum: 127881
+ Column 32: count: 9173 hasNull: true min: 0034fkcXMQI3 max: yyt0S8WorA sum: 149134
+ Column 33: count: 9173 hasNull: true min: 1969-12-31 13:59:30.929 max: 1969-12-31 14:00:30.808
+ Column 34: count: 9174 hasNull: true min: 1969-12-31 13:59:30.929 max: 1969-12-31 14:00:30.808
+ Column 35: count: 9174 hasNull: true true: 6138
+ Column 36: count: 9173 hasNull: true true: 3983
+
+File Statistics:
+ Column 0: count: 24576 hasNull: false
+ Column 1: count: 18346 hasNull: true min: -64 max: 62 sum: -79712
+ Column 2: count: 18348 hasNull: true min: -16379 max: 16376 sum: 14871980
+ Column 3: count: 18346 hasNull: true min: -1073279343 max: 1073680599 sum: 2876101727570
+ Column 4: count: 18346 hasNull: true min: -2147311592 max: 2145498388 sum: -3396920056818
+ Column 5: count: 18346 hasNull: true min: -64.0 max: 79.5530014038086 sum: -78959.27198576927
+ Column 6: count: 18348 hasNull: true min: -16379.0 max: 9763215.5639 sum: 1.12447306061E8
+ Column 7: count: 24576 hasNull: false min: 00020767-dd8f-4f4d-bd68-4b7be64b8e44 max: fffa3516-e219-4027-b0d3-72bb2e676c52 sum: 884736
+ Column 8: count: 24576 hasNull: false min: 000976f7-7075-4f3f-a564-5a375fafcc101416a2b7-7f64-41b7-851f-97d15405037e max: fffd0642-5f01-48cd-8d97-3428faee49e9b39f2b4c-efdc-4e5f-9ab5-4aa5394cb156 sum: 1769472
+ Column 9: count: 18346 hasNull: true min: 1969-12-31 13:59:30.929 max: 1969-12-31 14:00:30.808
+ Column 10: count: 18348 hasNull: true min: 1969-12-31 13:59:30.929 max: 1969-12-31 14:00:30.808
+ Column 11: count: 18348 hasNull: true true: 12276
+ Column 12: count: 18346 hasNull: true true: 7966
+ Column 13: count: 18346 hasNull: true min: -64 max: 62 sum: -79712
+ Column 14: count: 18348 hasNull: true min: -16379 max: 16376 sum: 14871980
+ Column 15: count: 18346 hasNull: true min: -1073279343 max: 1073680599 sum: 2876101727570
+ Column 16: count: 18346 hasNull: true min: -2147311592 max: 2145498388 sum: -3396920056818
+ Column 17: count: 18346 hasNull: true min: -64.0 max: 79.5530014038086 sum: -78959.27198576927
+ Column 18: count: 18348 hasNull: true min: -16379.0 max: 9763215.5639 sum: 1.12447306061E8
+ Column 19: count: 18348 hasNull: true min: 0042l0d5rPD6sMlJ7Ue0q max: yy2GiGM sum: 255762
+ Column 20: count: 18346 hasNull: true min: 0034fkcXMQI3 max: yyt0S8WorA sum: 298268
+ Column 21: count: 18346 hasNull: true min: 1969-12-31 13:59:30.929 max: 1969-12-31 14:00:30.808
+ Column 22: count: 18348 hasNull: true min: 1969-12-31 13:59:30.929 max: 1969-12-31 14:00:30.808
+ Column 23: count: 18348 hasNull: true true: 12276
+ Column 24: count: 18346 hasNull: true true: 7966
+ Column 25: count: 18346 hasNull: true min: -64 max: 62 sum: -79712
+ Column 26: count: 18348 hasNull: true min: -16379 max: 16376 sum: 14871980
+ Column 27: count: 18346 hasNull: true min: -1073279343 max: 1073680599 sum: 2876101727570
+ Column 28: count: 18346 hasNull: true min: -2147311592 max: 2145498388 sum: -3396920056818
+ Column 29: count: 18346 hasNull: true min: -64.0 max: 79.5530014038086 sum: -78959.27198576927
+ Column 30: count: 18348 hasNull: true min: -16379.0 max: 9763215.5639 sum: 1.12447306061E8
+ Column 31: count: 18348 hasNull: true min: 0042l0d5rPD6sMlJ7Ue0q max: yy2GiGM sum: 255762
+ Column 32: count: 18346 hasNull: true min: 0034fkcXMQI3 max: yyt0S8WorA sum: 298268
+ Column 33: count: 18346 hasNull: true min: 1969-12-31 13:59:30.929 max: 1969-12-31 14:00:30.808
+ Column 34: count: 18348 hasNull: true min: 1969-12-31 13:59:30.929 max: 1969-12-31 14:00:30.808
+ Column 35: count: 18348 hasNull: true true: 12276
+ Column 36: count: 18346 hasNull: true true: 7966
+
+Stripes:
+ Stripe: offset: 3 data: 1500017 rows: 12288 tail: 501 index: 2836
+ Stream: column 0 section ROW_INDEX start: 3 length 21
+ Stream: column 1 section ROW_INDEX start: 24 length 53
+ Stream: column 2 section ROW_INDEX start: 77 length 67
+ Stream: column 3 section ROW_INDEX start: 144 length 81
+ Stream: column 4 section ROW_INDEX start: 225 length 83
+ Stream: column 5 section ROW_INDEX start: 308 length 77
+ Stream: column 6 section ROW_INDEX start: 385 length 77
+ Stream: column 7 section ROW_INDEX start: 462 length 176
+ Stream: column 8 section ROW_INDEX start: 638 length 267
+ Stream: column 9 section ROW_INDEX start: 905 length 63
+ Stream: column 10 section ROW_INDEX start: 968 length 57
+ Stream: column 11 section ROW_INDEX start: 1025 length 47
+ Stream: column 12 section ROW_INDEX start: 1072 length 47
+ Stream: column 13 section ROW_INDEX start: 1119 length 53
+ Stream: column 14 section ROW_INDEX start: 1172 length 67
+ Stream: column 15 section ROW_INDEX start: 1239 length 81
+ Stream: column 16 section ROW_INDEX start: 1320 length 83
+ Stream: column 17 section ROW_INDEX start: 1403 length 77
+ Stream: column 18 section ROW_INDEX start: 1480 length 77
+ Stream: column 19 section ROW_INDEX start: 1557 length 115
+ Stream: column 20 section ROW_INDEX start: 1672 length 93
+ Stream: column 21 section ROW_INDEX start: 1765 length 63
+ Stream: column 22 section ROW_INDEX start: 1828 length 57
+ Stream: column 23 section ROW_INDEX start: 1885 length 47
+ Stream: column 24 section ROW_INDEX start: 1932 length 47
+ Stream: column 25 section ROW_INDEX start: 1979 length 53
+ Stream: column 26 section ROW_INDEX start: 2032 length 67
+ Stream: column 27 section ROW_INDEX start: 2099 length 81
+ Stream: column 28 section ROW_INDEX start: 2180 length 83
+ Stream: column 29 section ROW_INDEX start: 2263 length 77
+ Stream: column 30 section ROW_INDEX start: 2340 length 77
+ Stream: column 31 section ROW_INDEX start: 2417 length 115
+ Stream: column 32 section ROW_INDEX start: 2532 length 93
+ Stream: column 33 section ROW_INDEX start: 2625 length 63
+ Stream: column 34 section ROW_INDEX start: 2688 length 57
+ Stream: column 35 section ROW_INDEX start: 2745 length 47
+ Stream: column 36 section ROW_INDEX start: 2792 length 47
+ Stream: column 1 section PRESENT start: 2839 length 51
+ Stream: column 1 section DATA start: 2890 length 5448
+ Stream: column 2 section PRESENT start: 8338 length 53
+ Stream: column 2 section DATA start: 8391 length 12144
+ Stream: column 3 section PRESENT start: 20535 length 53
+ Stream: column 3 section DATA start: 20588 length 24618
+ Stream: column 4 section PRESENT start: 45206 length 52
+ Stream: column 4 section DATA start: 45258 length 24681
+ Stream: column 5 section PRESENT start: 69939 length 51
+ Stream: column 5 section DATA start: 69990 length 9927
+ Stream: column 6 section PRESENT start: 79917 length 53
+ Stream: column 6 section DATA start: 79970 length 19755
+ Stream: column 7 section DATA start: 99725 length 258570
+ Stream: column 7 section LENGTH start: 358295 length 108
+ Stream: column 8 section DATA start: 358403 length 517341
+ Stream: column 8 section LENGTH start: 875744 length 108
+ Stream: column 9 section PRESENT start: 875852 length 52
+ Stream: column 9 section DATA start: 875904 length 8045
+ Stream: column 9 section SECONDARY start: 883949 length 9555
+ Stream: column 10 section PRESENT start: 893504 length 58
+ Stream: column 10 section DATA start: 893562 length 8082
+ Stream: column 10 section SECONDARY start: 901644 length 9590
+ Stream: column 11 section PRESENT start: 911234 length 51
+ Stream: column 11 section DATA start: 911285 length 782
+ Stream: column 12 section PRESENT start: 912067 length 54
+ Stream: column 12 section DATA start: 912121 length 783
+ Stream: column 13 section PRESENT start: 912904 length 51
+ Stream: column 13 section DATA start: 912955 length 5448
+ Stream: column 14 section PRESENT start: 918403 length 53
+ Stream: column 14 section DATA start: 918456 length 12144
+ Stream: column 15 section PRESENT start: 930600 length 53
+ Stream: column 15 section DATA start: 930653 length 24618
+ Stream: column 16 section PRESENT start: 955271 length 52
+ Stream: column 16 section DATA start: 955323 length 24681
+ Stream: column 17 section PRESENT start: 980004 length 51
+ Stream: column 17 section DATA start: 980055 length 9927
+ Stream: column 18 section PRESENT start: 989982 length 53
+ Stream: column 18 section DATA start: 990035 length 19755
+ Stream: column 19 section PRESENT start: 1009790 length 51
+ Stream: column 19 section DATA start: 1009841 length 11009
+ Stream: column 19 section LENGTH start: 1020850 length 3722
+ Stream: column 19 section DICTIONARY_DATA start: 1024572 length 65435
+ Stream: column 20 section PRESENT start: 1090007 length 54
+ Stream: column 20 section DATA start: 1090061 length 11006
+ Stream: column 20 section LENGTH start: 1101067 length 3739
+ Stream: column 20 section DICTIONARY_DATA start: 1104806 length 66022
+ Stream: column 21 section PRESENT start: 1170828 length 52
+ Stream: column 21 section DATA start: 1170880 length 8045
+ Stream: column 21 section SECONDARY start: 1178925 length 9555
+ Stream: column 22 section PRESENT start: 1188480 length 58
+ Stream: column 22 section DATA start: 1188538 length 8082
+ Stream: column 22 section SECONDARY start: 1196620 length 9590
+ Stream: column 23 section PRESENT start: 1206210 length 51
+ Stream: column 23 section DATA start: 1206261 length 782
+ Stream: column 24 section PRESENT start: 1207043 length 54
+ Stream: column 24 section DATA start: 1207097 length 783
+ Stream: column 25 section PRESENT start: 1207880 length 51
+ Stream: column 25 section DATA start: 1207931 length 5448
+ Stream: column 26 section PRESENT start: 1213379 length 53
+ Stream: column 26 section DATA start: 1213432 length 12144
+ Stream: column 27 section PRESENT start: 1225576 length 53
+ Stream: column 27 section DATA start: 1225629 length 24618
+ Stream: column 28 section PRESENT start: 1250247 length 52
+ Stream: column 28 section DATA start: 1250299 length 24681
+ Stream: column 29 section PRESENT start: 1274980 length 51
+ Stream: column 29 section DATA start: 1275031 length 9927
+ Stream: column 30 section PRESENT start: 1284958 length 53
+ Stream: column 30 section DATA start: 1285011 length 19755
+ Stream: column 31 section PRESENT start: 1304766 length 51
+ Stream: column 31 section DATA start: 1304817 length 11009
+ Stream: column 31 section LENGTH start: 1315826 length 3722
+ Stream: column 31 section DICTIONARY_DATA start: 1319548 length 65435
+ Stream: column 32 section PRESENT start: 1384983 length 54
+ Stream: column 32 section DATA start: 1385037 length 11006
+ Stream: column 32 section LENGTH start: 1396043 length 3739
+ Stream: column 32 section DICTIONARY_DATA start: 1399782 length 66022
+ Stream: column 33 section PRESENT start: 1465804 length 52
+ Stream: column 33 section DATA start: 1465856 length 8045
+ Stream: column 33 section SECONDARY start: 1473901 length 9555
+ Stream: column 34 section PRESENT start: 1483456 length 58
+ Stream: column 34 section DATA start: 1483514 length 8082
+ Stream: column 34 section SECONDARY start: 1491596 length 9590
+ Stream: column 35 section PRESENT start: 1501186 length 51
+ Stream: column 35 section DATA start: 1501237 length 782
+ Stream: column 36 section PRESENT start: 1502019 length 54
+ Stream: column 36 section DATA start: 1502073 length 783
+ Encoding column 0: DIRECT
+ Encoding column 1: DIRECT
+ Encoding column 2: DIRECT_V2
+ Encoding column 3: DIRECT_V2
+ Encoding column 4: DIRECT_V2
+ Encoding column 5: DIRECT
+ Encoding column 6: DIRECT
+ Encoding column 7: DIRECT_V2
+ Encoding column 8: DIRECT_V2
+ Encoding column 9: DIRECT_V2
+ Encoding column 10: DIRECT_V2
+ Encoding column 11: DIRECT
+ Encoding column 12: DIRECT
+ Encoding column 13: DIRECT
+ Encoding column 14: DIRECT_V2
+ Encoding column 15: DIRECT_V2
+ Encoding column 16: DIRECT_V2
+ Encoding column 17: DIRECT
+ Encoding column 18: DIRECT
+ Encoding column 19: DICTIONARY_V2[6083]
+ Encoding column 20: DICTIONARY_V2[6081]
+ Encoding column 21: DIRECT_V2
+ Encoding column 22: DIRECT_V2
+ Encoding column 23: DIRECT
+ Encoding column 24: DIRECT
+ Encoding column 25: DIRECT
+ Encoding column 26: DIRECT_V2
+ Encoding column 27: DIRECT_V2
+ Encoding column 28: DIRECT_V2
+ Encoding column 29: DIRECT
+ Encoding column 30: DIRECT
+ Encoding column 31: DICTIONARY_V2[6083]
+ Encoding column 32: DICTIONARY_V2[6081]
+ Encoding column 33: DIRECT_V2
+ Encoding column 34: DIRECT_V2
+ Encoding column 35: DIRECT
+ Encoding column 36: DIRECT
+ Row group indices for column 1:
+ Entry 0: count: 7909 hasNull: true min: -64 max: 62 sum: -50203 positions: 0,0,0,0,0,0,0
+ Entry 1: count: 1264 hasNull: true min: -64 max: 62 sum: 10347 positions: 0,182,99,0,0,5937,2
+ Stripe: offset: 1503357 data: 1500017 rows: 12288 tail: 501 index: 2836
+ Stream: column 0 section ROW_INDEX start: 1503357 length 21
+ Stream: column 1 section ROW_INDEX start: 1503378 length 53
+ Stream: column 2 section ROW_INDEX start: 1503431 length 67
+ Stream: column 3 section ROW_INDEX start: 1503498 length 81
+ Stream: column 4 section ROW_INDEX start: 1503579 length 83
+ Stream: column 5 section ROW_INDEX start: 1503662 length 77
+ Stream: column 6 section ROW_INDEX start: 1503739 length 77
+ Stream: column 7 section ROW_INDEX start: 1503816 length 176
+ Stream: column 8 section ROW_INDEX start: 1503992 length 267
+ Stream: column 9 section ROW_INDEX start: 1504259 length 63
+ Stream: column 10 section ROW_INDEX start: 1504322 length 57
+ Stream: column 11 section ROW_INDEX start: 1504379 length 47
+ Stream: column 12 section ROW_INDEX start: 1504426 length 47
+ Stream: column 13 section ROW_INDEX start: 1504473 length 53
+ Stream: column 14 section ROW_INDEX start: 1504526 length 67
+ Stream: column 15 section ROW_INDEX start: 1504593 length 81
+ Stream: column 16 section ROW_INDEX start: 1504674 length 83
+ Stream: column 17 section ROW_INDEX start: 1504757 length 77
+ Stream: column 18 section ROW_INDEX start: 1504834 length 77
+ Stream: column 19 section ROW_INDEX start: 1504911 length 115
+ Stream: column 20 section ROW_INDEX start: 1505026 length 93
+ Stream: column 21 section ROW_INDEX start: 1505119 length 63
+ Stream: column 22 section ROW_INDEX start: 1505182 length 57
+ Stream: column 23 section ROW_INDEX start: 1505239 length 47
+ Stream: column 24 section ROW_INDEX start: 1505286 length 47
+ Stream: column 25 section ROW_INDEX start: 1505333 length 53
+ Stream: column 26 section ROW_INDEX start: 1505386 length 67
+ Stream: column 27 section ROW_INDEX start: 1505453 length 81
+ Stream: column 28 section ROW_INDEX start: 1505534 length 83
+ Stream: column 29 section ROW_INDEX start: 1505617 length 77
+ Stream: column 30 section ROW_INDEX start: 1505694 length 77
+ Stream: column 31 section ROW_INDEX start: 1505771 length 115
+ Stream: column 32 section ROW_INDEX start: 1505886 length 93
+ Stream: column 33 section ROW_INDEX start: 1505979 length 63
+ Stream: column 34 section ROW_INDEX start: 1506042 length 57
+ Stream: column 35 section ROW_INDEX start: 1506099 length 47
+ Stream: column 36 section ROW_INDEX start: 1506146 length 47
+ Stream: column 1 section PRESENT start: 1506193 length 51
+ Stream: column 1 section DATA start: 1506244 length 5448
+ Stream: column 2 section PRESENT start: 1511692 length 53
+ Stream: column 2 section DATA start: 1511745 length 12144
+ Stream: column 3 section PRESENT start: 1523889 length 53
+ Stream: column 3 section DATA start: 1523942 length 24618
+ Stream: column 4 section PRESENT start: 1548560 length 52
+ Stream: column 4 section DATA start: 1548612 length 24681
+ Stream: column 5 section PRESENT start: 1573293 length 51
+ Stream: column 5 section DATA start: 1573344 length 9927
+ Stream: column 6 section PRESENT start: 1583271 length 53
+ Stream: column 6 section DATA start: 1583324 length 19755
+ Stream: column 7 section DATA start: 1603079 length 258570
+ Stream: column 7 section LENGTH start: 1861649 length 108
+ Stream: column 8 section DATA start: 1861757 length 517341
+ Stream: column 8 section LENGTH start: 2379098 length 108
+ Stream: column 9 section PRESENT start: 2379206 length 52
+ Stream: column 9 section DATA start: 2379258 length 8045
+ Stream: column 9 section SECONDARY start: 2387303 length 9555
+ Stream: column 10 section PRESENT start: 2396858 length 58
+ Stream: column 10 section DATA start: 2396916 length 8082
+ Stream: column 10 section SECONDARY start: 2404998 length 9590
+ Stream: column 11 section PRESENT start: 2414588 length 51
+ Stream: column 11 section DATA start: 2414639 length 782
+ Stream: column 12 section PRESENT start: 2415421 length 54
+ Stream: column 12 section DATA start: 2415475 length 783
+ Stream: column 13 section PRESENT start: 2416258 length 51
+ Stream: column 13 section DATA start: 2416309 length 5448
+ Stream: column 14 section PRESENT start: 2421757 length 53
+ Stream: column 14 section DATA start: 2421810 length 12144
+ Stream: column 15 section PRESENT start: 2433954 length 53
+ Stream: column 15 section DATA start: 2434007 length 24618
+ Stream: column 16 section PRESENT start: 2458625 length 52
+ Stream: column 16 section DATA start: 2458677 length 24681
+ Stream: column 17 section PRESENT start: 2483358 length 51
+ Stream: column 17 section DATA start: 2483409 length 9927
+ Stream: column 18 section PRESENT start: 2493336 length 53
+ Stream: column 18 section DATA start: 2493389 length 19755
+ Stream: column 19 section PRESENT start: 2513144 length 51
+ Stream: column 19 section DATA start: 2513195 length 11009
+ Stream: column 19 section LENGTH start: 2524204 length 3722
+ Stream: column 19 section DICTIONARY_DATA start: 2527926 length 65435
+ Stream: column 20 section PRESENT start: 2593361 length 54
+ Stream: column 20 section DATA start: 2593415 length 11006
+ Stream: column 20 section LENGTH start: 2604421 length 3739
+ Stream: column 20 section DICTIONARY_DATA start: 2608160 length 66022
+ Stream: column 21 section PRESENT start: 2674182 length 52
+ Stream: column 21 section DATA start: 2674234 length 8045
+ Stream: column 21 section SECONDARY start: 2682279 length 9555
+ Stream: column 22 section PRESENT start: 2691834 length 58
+ Stream: column 22 section DATA start: 2691892 length 8082
+ Stream: column 22 section SECONDARY start: 2699974 length 9590
+ Stream: column 23 section PRESENT start: 2709564 length 51
+ Stream: column 23 section DATA start: 2709615 length 782
+ Stream: column 24 section PRESENT start: 2710397 length 54
+ Stream: column 24 section DATA start: 2710451 length 783
+ Stream: column 25 section PRESENT start: 2711234 length 51
+ Stream: column 25 section DATA start: 2711285 length 5448
+ Stream: column 26 section PRESENT start: 2716733 length 53
+ Stream: column 26 section DATA start: 2716786 length 12144
+ Stream: column 27 section PRESENT start: 2728930 length 53
+ Stream: column 27 section DATA start: 2728983 length 24618
+ Stream: column 28 section PRESENT start: 2753601 length 52
+ Stream: column 28 section DATA start: 2753653 length 24681
+ Stream: column 29 section PRESENT start: 2778334 length 51
+ Stream: column 29 section DATA start: 2778385 length 9927
+ Stream: column 30 section PRESENT start: 2788312 length 53
+ Stream: column 30 section DATA start: 2788365 length 19755
+ Stream: column 31 section PRESENT start: 2808120 length 51
+ Stream: column 31 section DATA start: 2808171 length 11009
+ Stream: column 31 section LENGTH start: 2819180 length 3722
+ Stream: column 31 section DICTIONARY_DATA start: 2822902 length 65435
+ Stream: column 32 section PRESENT start: 2888337 length 54
+ Stream: column 32 section DATA start: 2888391 length 11006
+ Stream: column 32 section LENGTH start: 2899397 length 3739
+ Stream: column 32 section DICTIONARY_DATA start: 2903136 length 66022
+ Stream: column 33 section PRESENT start: 2969158 length 52
+ Stream: column 33 section DATA start: 2969210 length 8045
+ Stream: column 33 section SECONDARY start: 2977255 length 9555
+ Stream: column 34 section PRESENT start: 2986810 length 58
+ Stream: column 34 section DATA start: 2986868 length 8082
+ Stream: column 34 section SECONDARY start: 2994950 length 9590
+ Stream: column 35 section PRESENT start: 3004540 length 51
+ Stream: column 35 section DATA start: 3004591 length 782
+ Stream: column 36 section PRESENT start: 3005373 length 54
+ Stream: column 36 section DATA start: 3005427 length 783
+ Encoding column 0: DIRECT
+ Encoding column 1: DIRECT
+ Encoding column 2: DIRECT_V2
+ Encoding column 3: DIRECT_V2
+ Encoding column 4: DIRECT_V2
+ Encoding column 5: DIRECT
+ Encoding column 6: DIRECT
+ Encoding column 7: DIRECT_V2
+ Encoding column 8: DIRECT_V2
+ Encoding column 9: DIRECT_V2
+ Encoding column 10: DIRECT_V2
+ Encoding column 11: DIRECT
+ Encoding column 12: DIRECT
+ Encoding column 13: DIRECT
+ Encoding column 14: DIRECT_V2
+ Encoding column 15: DIRECT_V2
+ Encoding column 16: DIRECT_V2
+ Encoding column 17: DIRECT
+ Encoding column 18: DIRECT
+ Encoding column 19: DICTIONARY_V2[6083]
+ Encoding column 20: DICTIONARY_V2[6081]
+ Encoding column 21: DIRECT_V2
+ Encoding column 22: DIRECT_V2
+ Encoding column 23: DIRECT
+ Encoding column 24: DIRECT
+ Encoding column 25: DIRECT
+ Encoding column 26: DIRECT_V2
+ Encoding column 27: DIRECT_V2
+ Encoding column 28: DIRECT_V2
+ Encoding column 29: DIRECT
+ Encoding column 30: DIRECT
+ Encoding column 31: DICTIONARY_V2[6083]
+ Encoding column 32: DICTIONARY_V2[6081]
+ Encoding column 33: DIRECT_V2
+ Encoding column 34: DIRECT_V2
+ Encoding column 35: DIRECT
+ Encoding column 36: DIRECT
+ Row group indices for column 1:
+ Entry 0: count: 7909 hasNull: true min: -64 max: 62 sum: -50203 positions: 0,0,0,0,0,0,0
+ Entry 1: count: 1264 hasNull: true min: -64 max: 62 sum: 10347 positions: 0,182,99,0,0,5937,2
+
+File length: 3007981 bytes
+Padding length: 0 bytes
+Padding ratio: 0%
+________________________________________________________________________________________________________________________
+
+-- END ORC FILE DUMP --
+-50 -13326 528534767 NULL -50.0 -13326.0 18f2de7d-0c69-4052-9c4b-64f196d6d589 9edba9e8-1f91-47e7-b31f-451eeb5feb7781ba3a3e-c22e-4412-91a2-86ad05ae1ca8 1969-12-31 15:59:46.674 1969-12-31 16:00:07.875 true NULL -50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:07.875 true NULL -50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:07.875 true NULL
http://git-wip-us.apache.org/repos/asf/hive/blob/8c1f055d/ql/src/test/results/clientpositive/tez/orc_merge12.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/orc_merge12.q.out b/ql/src/test/results/clientpositive/tez/orc_merge12.q.out
new file mode 100644
index 0000000..f23be5a
--- /dev/null
+++ b/ql/src/test/results/clientpositive/tez/orc_merge12.q.out
@@ -0,0 +1,606 @@
+PREHOOK: query: CREATE TABLE `alltypesorc3xcols`(
+ `atinyint` tinyint,
+ `asmallint` smallint,
+ `aint` int,
+ `abigint` bigint,
+ `afloat` float,
+ `adouble` double,
+ `astring1` string,
+ `astring2` string,
+ `atimestamp1` timestamp,
+ `atimestamp2` timestamp,
+ `aboolean1` boolean,
+ `aboolean2` boolean,
+ `btinyint` tinyint,
+ `bsmallint` smallint,
+ `bint` int,
+ `bbigint` bigint,
+ `bfloat` float,
+ `bdouble` double,
+ `bstring1` string,
+ `bstring2` string,
+ `btimestamp1` timestamp,
+ `btimestamp2` timestamp,
+ `bboolean1` boolean,
+ `bboolean2` boolean,
+ `ctinyint` tinyint,
+ `csmallint` smallint,
+ `cint` int,
+ `cbigint` bigint,
+ `cfloat` float,
+ `cdouble` double,
+ `cstring1` string,
+ `cstring2` string,
+ `ctimestamp1` timestamp,
+ `ctimestamp2` timestamp,
+ `cboolean1` boolean,
+ `cboolean2` boolean) stored as ORC
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@alltypesorc3xcols
+POSTHOOK: query: CREATE TABLE `alltypesorc3xcols`(
+ `atinyint` tinyint,
+ `asmallint` smallint,
+ `aint` int,
+ `abigint` bigint,
+ `afloat` float,
+ `adouble` double,
+ `astring1` string,
+ `astring2` string,
+ `atimestamp1` timestamp,
+ `atimestamp2` timestamp,
+ `aboolean1` boolean,
+ `aboolean2` boolean,
+ `btinyint` tinyint,
+ `bsmallint` smallint,
+ `bint` int,
+ `bbigint` bigint,
+ `bfloat` float,
+ `bdouble` double,
+ `bstring1` string,
+ `bstring2` string,
+ `btimestamp1` timestamp,
+ `btimestamp2` timestamp,
+ `bboolean1` boolean,
+ `bboolean2` boolean,
+ `ctinyint` tinyint,
+ `csmallint` smallint,
+ `cint` int,
+ `cbigint` bigint,
+ `cfloat` float,
+ `cdouble` double,
+ `cstring1` string,
+ `cstring2` string,
+ `ctimestamp1` timestamp,
+ `ctimestamp2` timestamp,
+ `cboolean1` boolean,
+ `cboolean2` boolean) stored as ORC
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@alltypesorc3xcols
+PREHOOK: query: load data local inpath '../../data/files/alltypesorc3xcols' into table alltypesorc3xcols
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@alltypesorc3xcols
+POSTHOOK: query: load data local inpath '../../data/files/alltypesorc3xcols' into table alltypesorc3xcols
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@alltypesorc3xcols
+PREHOOK: query: load data local inpath '../../data/files/alltypesorc3xcols' into table alltypesorc3xcols
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@alltypesorc3xcols
+POSTHOOK: query: load data local inpath '../../data/files/alltypesorc3xcols' into table alltypesorc3xcols
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@alltypesorc3xcols
+PREHOOK: query: select count(*) from alltypesorc3xcols
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc3xcols
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from alltypesorc3xcols
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc3xcols
+#### A masked pattern was here ####
+24576
+PREHOOK: query: select sum(hash(*)) from alltypesorc3xcols
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc3xcols
+#### A masked pattern was here ####
+POSTHOOK: query: select sum(hash(*)) from alltypesorc3xcols
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc3xcols
+#### A masked pattern was here ####
+-302946892512
+PREHOOK: query: alter table alltypesorc3xcols concatenate
+PREHOOK: type: ALTER_TABLE_MERGE
+PREHOOK: Input: default@alltypesorc3xcols
+PREHOOK: Output: default@alltypesorc3xcols
+POSTHOOK: query: alter table alltypesorc3xcols concatenate
+POSTHOOK: type: ALTER_TABLE_MERGE
+POSTHOOK: Input: default@alltypesorc3xcols
+POSTHOOK: Output: default@alltypesorc3xcols
+PREHOOK: query: select count(*) from alltypesorc3xcols
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc3xcols
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from alltypesorc3xcols
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc3xcols
+#### A masked pattern was here ####
+24576
+PREHOOK: query: select sum(hash(*)) from alltypesorc3xcols
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc3xcols
+#### A masked pattern was here ####
+POSTHOOK: query: select sum(hash(*)) from alltypesorc3xcols
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc3xcols
+#### A masked pattern was here ####
+-302946892512
+PREHOOK: query: select * from alltypesorc3xcols limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc3xcols
+#### A masked pattern was here ####
+-- BEGIN ORC FILE DUMP --
+#### A masked pattern was here ####
+File Version: 0.12 with HIVE_13083
+Rows: 24576
+Compression: ZLIB
+Compression size: 262144
+Type: struct<_col0:tinyint,_col1:smallint,_col2:int,_col3:bigint,_col4:float,_col5:double,_col6:string,_col7:string,_col8:timestamp,_col9:timestamp,_col10:boolean,_col11:boolean,_col12:tinyint,_col13:smallint,_col14:int,_col15:bigint,_col16:float,_col17:double,_col18:string,_col19:string,_col20:timestamp,_col21:timestamp,_col22:boolean,_col23:boolean,_col24:tinyint,_col25:smallint,_col26:int,_col27:bigint,_col28:float,_col29:double,_col30:string,_col31:string,_col32:timestamp,_col33:timestamp,_col34:boolean,_col35:boolean>
+
+Stripe Statistics:
+ Stripe 1:
+ Column 0: count: 12288 hasNull: false
+ Column 1: count: 9173 hasNull: true min: -64 max: 62 sum: -39856
+ Column 2: count: 9174 hasNull: true min: -16379 max: 16376 sum: 7435990
+ Column 3: count: 9173 hasNull: true min: -1073279343 max: 1073680599 sum: 1438050863785
+ Column 4: count: 9173 hasNull: true min: -2147311592 max: 2145498388 sum: -1698460028409
+ Column 5: count: 9173 hasNull: true min: -64.0 max: 79.5530014038086 sum: -39479.635992884636
+ Column 6: count: 9174 hasNull: true min: -16379.0 max: 9763215.5639 sum: 5.62236530305E7
+ Column 7: count: 12288 hasNull: false min: 00020767-dd8f-4f4d-bd68-4b7be64b8e44 max: fffa3516-e219-4027-b0d3-72bb2e676c52 sum: 442368
+ Column 8: count: 12288 hasNull: false min: 000976f7-7075-4f3f-a564-5a375fafcc101416a2b7-7f64-41b7-851f-97d15405037e max: fffd0642-5f01-48cd-8d97-3428faee49e9b39f2b4c-efdc-4e5f-9ab5-4aa5394cb156 sum: 884736
+ Column 9: count: 9173 hasNull: true min: 1969-12-31 13:59:30.929 max: 1969-12-31 14:00:30.808
+ Column 10: count: 9174 hasNull: true min: 1969-12-31 13:59:30.929 max: 1969-12-31 14:00:30.808
+ Column 11: count: 9174 hasNull: true true: 6138
+ Column 12: count: 9173 hasNull: true true: 3983
+ Column 13: count: 9173 hasNull: true min: -64 max: 62 sum: -39856
+ Column 14: count: 9174 hasNull: true min: -16379 max: 16376 sum: 7435990
+ Column 15: count: 9173 hasNull: true min: -1073279343 max: 1073680599 sum: 1438050863785
+ Column 16: count: 9173 hasNull: true min: -2147311592 max: 2145498388 sum: -1698460028409
+ Column 17: count: 9173 hasNull: true min: -64.0 max: 79.5530014038086 sum: -39479.635992884636
+ Column 18: count: 9174 hasNull: true min: -16379.0 max: 9763215.5639 sum: 5.62236530305E7
+ Column 19: count: 9174 hasNull: true min: 0042l0d5rPD6sMlJ7Ue0q max: yy2GiGM sum: 127881
+ Column 20: count: 9173 hasNull: true min: 0034fkcXMQI3 max: yyt0S8WorA sum: 149134
+ Column 21: count: 9173 hasNull: true min: 1969-12-31 13:59:30.929 max: 1969-12-31 14:00:30.808
+ Column 22: count: 9174 hasNull: true min: 1969-12-31 13:59:30.929 max: 1969-12-31 14:00:30.808
+ Column 23: count: 9174 hasNull: true true: 6138
+ Column 24: count: 9173 hasNull: true true: 3983
+ Column 25: count: 9173 hasNull: true min: -64 max: 62 sum: -39856
+ Column 26: count: 9174 hasNull: true min: -16379 max: 16376 sum: 7435990
+ Column 27: count: 9173 hasNull: true min: -1073279343 max: 1073680599 sum: 1438050863785
+ Column 28: count: 9173 hasNull: true min: -2147311592 max: 2145498388 sum: -1698460028409
+ Column 29: count: 9173 hasNull: true min: -64.0 max: 79.5530014038086 sum: -39479.635992884636
+ Column 30: count: 9174 hasNull: true min: -16379.0 max: 9763215.5639 sum: 5.62236530305E7
+ Column 31: count: 9174 hasNull: true min: 0042l0d5rPD6sMlJ7Ue0q max: yy2GiGM sum: 127881
+ Column 32: count: 9173 hasNull: true min: 0034fkcXMQI3 max: yyt0S8WorA sum: 149134
+ Column 33: count: 9173 hasNull: true min: 1969-12-31 13:59:30.929 max: 1969-12-31 14:00:30.808
+ Column 34: count: 9174 hasNull: true min: 1969-12-31 13:59:30.929 max: 1969-12-31 14:00:30.808
+ Column 35: count: 9174 hasNull: true true: 6138
+ Column 36: count: 9173 hasNull: true true: 3983
+ Stripe 2:
+ Column 0: count: 12288 hasNull: false
+ Column 1: count: 9173 hasNull: true min: -64 max: 62 sum: -39856
+ Column 2: count: 9174 hasNull: true min: -16379 max: 16376 sum: 7435990
+ Column 3: count: 9173 hasNull: true min: -1073279343 max: 1073680599 sum: 1438050863785
+ Column 4: count: 9173 hasNull: true min: -2147311592 max: 2145498388 sum: -1698460028409
+ Column 5: count: 9173 hasNull: true min: -64.0 max: 79.5530014038086 sum: -39479.635992884636
+ Column 6: count: 9174 hasNull: true min: -16379.0 max: 9763215.5639 sum: 5.62236530305E7
+ Column 7: count: 12288 hasNull: false min: 00020767-dd8f-4f4d-bd68-4b7be64b8e44 max: fffa3516-e219-4027-b0d3-72bb2e676c52 sum: 442368
+ Column 8: count: 12288 hasNull: false min: 000976f7-7075-4f3f-a564-5a375fafcc101416a2b7-7f64-41b7-851f-97d15405037e max: fffd0642-5f01-48cd-8d97-3428faee49e9b39f2b4c-efdc-4e5f-9ab5-4aa5394cb156 sum: 884736
+ Column 9: count: 9173 hasNull: true min: 1969-12-31 13:59:30.929 max: 1969-12-31 14:00:30.808
+ Column 10: count: 9174 hasNull: true min: 1969-12-31 13:59:30.929 max: 1969-12-31 14:00:30.808
+ Column 11: count: 9174 hasNull: true true: 6138
+ Column 12: count: 9173 hasNull: true true: 3983
+ Column 13: count: 9173 hasNull: true min: -64 max: 62 sum: -39856
+ Column 14: count: 9174 hasNull: true min: -16379 max: 16376 sum: 7435990
+ Column 15: count: 9173 hasNull: true min: -1073279343 max: 1073680599 sum: 1438050863785
+ Column 16: count: 9173 hasNull: true min: -2147311592 max: 2145498388 sum: -1698460028409
+ Column 17: count: 9173 hasNull: true min: -64.0 max: 79.5530014038086 sum: -39479.635992884636
+ Column 18: count: 9174 hasNull: true min: -16379.0 max: 9763215.5639 sum: 5.62236530305E7
+ Column 19: count: 9174 hasNull: true min: 0042l0d5rPD6sMlJ7Ue0q max: yy2GiGM sum: 127881
+ Column 20: count: 9173 hasNull: true min: 0034fkcXMQI3 max: yyt0S8WorA sum: 149134
+ Column 21: count: 9173 hasNull: true min: 1969-12-31 13:59:30.929 max: 1969-12-31 14:00:30.808
+ Column 22: count: 9174 hasNull: true min: 1969-12-31 13:59:30.929 max: 1969-12-31 14:00:30.808
+ Column 23: count: 9174 hasNull: true true: 6138
+ Column 24: count: 9173 hasNull: true true: 3983
+ Column 25: count: 9173 hasNull: true min: -64 max: 62 sum: -39856
+ Column 26: count: 9174 hasNull: true min: -16379 max: 16376 sum: 7435990
+ Column 27: count: 9173 hasNull: true min: -1073279343 max: 1073680599 sum: 1438050863785
+ Column 28: count: 9173 hasNull: true min: -2147311592 max: 2145498388 sum: -1698460028409
+ Column 29: count: 9173 hasNull: true min: -64.0 max: 79.5530014038086 sum: -39479.635992884636
+ Column 30: count: 9174 hasNull: true min: -16379.0 max: 9763215.5639 sum: 5.62236530305E7
+ Column 31: count: 9174 hasNull: true min: 0042l0d5rPD6sMlJ7Ue0q max: yy2GiGM sum: 127881
+ Column 32: count: 9173 hasNull: true min: 0034fkcXMQI3 max: yyt0S8WorA sum: 149134
+ Column 33: count: 9173 hasNull: true min: 1969-12-31 13:59:30.929 max: 1969-12-31 14:00:30.808
+ Column 34: count: 9174 hasNull: true min: 1969-12-31 13:59:30.929 max: 1969-12-31 14:00:30.808
+ Column 35: count: 9174 hasNull: true true: 6138
+ Column 36: count: 9173 hasNull: true true: 3983
+
+File Statistics:
+ Column 0: count: 24576 hasNull: false
+ Column 1: count: 18346 hasNull: true min: -64 max: 62 sum: -79712
+ Column 2: count: 18348 hasNull: true min: -16379 max: 16376 sum: 14871980
+ Column 3: count: 18346 hasNull: true min: -1073279343 max: 1073680599 sum: 2876101727570
+ Column 4: count: 18346 hasNull: true min: -2147311592 max: 2145498388 sum: -3396920056818
+ Column 5: count: 18346 hasNull: true min: -64.0 max: 79.5530014038086 sum: -78959.27198576927
+ Column 6: count: 18348 hasNull: true min: -16379.0 max: 9763215.5639 sum: 1.12447306061E8
+ Column 7: count: 24576 hasNull: false min: 00020767-dd8f-4f4d-bd68-4b7be64b8e44 max: fffa3516-e219-4027-b0d3-72bb2e676c52 sum: 884736
+ Column 8: count: 24576 hasNull: false min: 000976f7-7075-4f3f-a564-5a375fafcc101416a2b7-7f64-41b7-851f-97d15405037e max: fffd0642-5f01-48cd-8d97-3428faee49e9b39f2b4c-efdc-4e5f-9ab5-4aa5394cb156 sum: 1769472
+ Column 9: count: 18346 hasNull: true min: 1969-12-31 13:59:30.929 max: 1969-12-31 14:00:30.808
+ Column 10: count: 18348 hasNull: true min: 1969-12-31 13:59:30.929 max: 1969-12-31 14:00:30.808
+ Column 11: count: 18348 hasNull: true true: 12276
+ Column 12: count: 18346 hasNull: true true: 7966
+ Column 13: count: 18346 hasNull: true min: -64 max: 62 sum: -79712
+ Column 14: count: 18348 hasNull: true min: -16379 max: 16376 sum: 14871980
+ Column 15: count: 18346 hasNull: true min: -1073279343 max: 1073680599 sum: 2876101727570
+ Column 16: count: 18346 hasNull: true min: -2147311592 max: 2145498388 sum: -3396920056818
+ Column 17: count: 18346 hasNull: true min: -64.0 max: 79.5530014038086 sum: -78959.27198576927
+ Column 18: count: 18348 hasNull: true min: -16379.0 max: 9763215.5639 sum: 1.12447306061E8
+ Column 19: count: 18348 hasNull: true min: 0042l0d5rPD6sMlJ7Ue0q max: yy2GiGM sum: 255762
+ Column 20: count: 18346 hasNull: true min: 0034fkcXMQI3 max: yyt0S8WorA sum: 298268
+ Column 21: count: 18346 hasNull: true min: 1969-12-31 13:59:30.929 max: 1969-12-31 14:00:30.808
+ Column 22: count: 18348 hasNull: true min: 1969-12-31 13:59:30.929 max: 1969-12-31 14:00:30.808
+ Column 23: count: 18348 hasNull: true true: 12276
+ Column 24: count: 18346 hasNull: true true: 7966
+ Column 25: count: 18346 hasNull: true min: -64 max: 62 sum: -79712
+ Column 26: count: 18348 hasNull: true min: -16379 max: 16376 sum: 14871980
+ Column 27: count: 18346 hasNull: true min: -1073279343 max: 1073680599 sum: 2876101727570
+ Column 28: count: 18346 hasNull: true min: -2147311592 max: 2145498388 sum: -3396920056818
+ Column 29: count: 18346 hasNull: true min: -64.0 max: 79.5530014038086 sum: -78959.27198576927
+ Column 30: count: 18348 hasNull: true min: -16379.0 max: 9763215.5639 sum: 1.12447306061E8
+ Column 31: count: 18348 hasNull: true min: 0042l0d5rPD6sMlJ7Ue0q max: yy2GiGM sum: 255762
+ Column 32: count: 18346 hasNull: true min: 0034fkcXMQI3 max: yyt0S8WorA sum: 298268
+ Column 33: count: 18346 hasNull: true min: 1969-12-31 13:59:30.929 max: 1969-12-31 14:00:30.808
+ Column 34: count: 18348 hasNull: true min: 1969-12-31 13:59:30.929 max: 1969-12-31 14:00:30.808
+ Column 35: count: 18348 hasNull: true true: 12276
+ Column 36: count: 18346 hasNull: true true: 7966
+
+Stripes:
+ Stripe: offset: 3 data: 1500017 rows: 12288 tail: 501 index: 2836
+ Stream: column 0 section ROW_INDEX start: 3 length 21
+ Stream: column 1 section ROW_INDEX start: 24 length 53
+ Stream: column 2 section ROW_INDEX start: 77 length 67
+ Stream: column 3 section ROW_INDEX start: 144 length 81
+ Stream: column 4 section ROW_INDEX start: 225 length 83
+ Stream: column 5 section ROW_INDEX start: 308 length 77
+ Stream: column 6 section ROW_INDEX start: 385 length 77
+ Stream: column 7 section ROW_INDEX start: 462 length 176
+ Stream: column 8 section ROW_INDEX start: 638 length 267
+ Stream: column 9 section ROW_INDEX start: 905 length 63
+ Stream: column 10 section ROW_INDEX start: 968 length 57
+ Stream: column 11 section ROW_INDEX start: 1025 length 47
+ Stream: column 12 section ROW_INDEX start: 1072 length 47
+ Stream: column 13 section ROW_INDEX start: 1119 length 53
+ Stream: column 14 section ROW_INDEX start: 1172 length 67
+ Stream: column 15 section ROW_INDEX start: 1239 length 81
+ Stream: column 16 section ROW_INDEX start: 1320 length 83
+ Stream: column 17 section ROW_INDEX start: 1403 length 77
+ Stream: column 18 section ROW_INDEX start: 1480 length 77
+ Stream: column 19 section ROW_INDEX start: 1557 length 115
+ Stream: column 20 section ROW_INDEX start: 1672 length 93
+ Stream: column 21 section ROW_INDEX start: 1765 length 63
+ Stream: column 22 section ROW_INDEX start: 1828 length 57
+ Stream: column 23 section ROW_INDEX start: 1885 length 47
+ Stream: column 24 section ROW_INDEX start: 1932 length 47
+ Stream: column 25 section ROW_INDEX start: 1979 length 53
+ Stream: column 26 section ROW_INDEX start: 2032 length 67
+ Stream: column 27 section ROW_INDEX start: 2099 length 81
+ Stream: column 28 section ROW_INDEX start: 2180 length 83
+ Stream: column 29 section ROW_INDEX start: 2263 length 77
+ Stream: column 30 section ROW_INDEX start: 2340 length 77
+ Stream: column 31 section ROW_INDEX start: 2417 length 115
+ Stream: column 32 section ROW_INDEX start: 2532 length 93
+ Stream: column 33 section ROW_INDEX start: 2625 length 63
+ Stream: column 34 section ROW_INDEX start: 2688 length 57
+ Stream: column 35 section ROW_INDEX start: 2745 length 47
+ Stream: column 36 section ROW_INDEX start: 2792 length 47
+ Stream: column 1 section PRESENT start: 2839 length 51
+ Stream: column 1 section DATA start: 2890 length 5448
+ Stream: column 2 section PRESENT start: 8338 length 53
+ Stream: column 2 section DATA start: 8391 length 12144
+ Stream: column 3 section PRESENT start: 20535 length 53
+ Stream: column 3 section DATA start: 20588 length 24618
+ Stream: column 4 section PRESENT start: 45206 length 52
+ Stream: column 4 section DATA start: 45258 length 24681
+ Stream: column 5 section PRESENT start: 69939 length 51
+ Stream: column 5 section DATA start: 69990 length 9927
+ Stream: column 6 section PRESENT start: 79917 length 53
+ Stream: column 6 section DATA start: 79970 length 19755
+ Stream: column 7 section DATA start: 99725 length 258570
+ Stream: column 7 section LENGTH start: 358295 length 108
+ Stream: column 8 section DATA start: 358403 length 517341
+ Stream: column 8 section LENGTH start: 875744 length 108
+ Stream: column 9 section PRESENT start: 875852 length 52
+ Stream: column 9 section DATA start: 875904 length 8045
+ Stream: column 9 section SECONDARY start: 883949 length 9555
+ Stream: column 10 section PRESENT start: 893504 length 58
+ Stream: column 10 section DATA start: 893562 length 8082
+ Stream: column 10 section SECONDARY start: 901644 length 9590
+ Stream: column 11 section PRESENT start: 911234 length 51
+ Stream: column 11 section DATA start: 911285 length 782
+ Stream: column 12 section PRESENT start: 912067 length 54
+ Stream: column 12 section DATA start: 912121 length 783
+ Stream: column 13 section PRESENT start: 912904 length 51
+ Stream: column 13 section DATA start: 912955 length 5448
+ Stream: column 14 section PRESENT start: 918403 length 53
+ Stream: column 14 section DATA start: 918456 length 12144
+ Stream: column 15 section PRESENT start: 930600 length 53
+ Stream: column 15 section DATA start: 930653 length 24618
+ Stream: column 16 section PRESENT start: 955271 length 52
+ Stream: column 16 section DATA start: 955323 length 24681
+ Stream: column 17 section PRESENT start: 980004 length 51
+ Stream: column 17 section DATA start: 980055 length 9927
+ Stream: column 18 section PRESENT start: 989982 length 53
+ Stream: column 18 section DATA start: 990035 length 19755
+ Stream: column 19 section PRESENT start: 1009790 length 51
+ Stream: column 19 section DATA start: 1009841 length 11009
+ Stream: column 19 section LENGTH start: 1020850 length 3722
+ Stream: column 19 section DICTIONARY_DATA start: 1024572 length 65435
+ Stream: column 20 section PRESENT start: 1090007 length 54
+ Stream: column 20 section DATA start: 1090061 length 11006
+ Stream: column 20 section LENGTH start: 1101067 length 3739
+ Stream: column 20 section DICTIONARY_DATA start: 1104806 length 66022
+ Stream: column 21 section PRESENT start: 1170828 length 52
+ Stream: column 21 section DATA start: 1170880 length 8045
+ Stream: column 21 section SECONDARY start: 1178925 length 9555
+ Stream: column 22 section PRESENT start: 1188480 length 58
+ Stream: column 22 section DATA start: 1188538 length 8082
+ Stream: column 22 section SECONDARY start: 1196620 length 9590
+ Stream: column 23 section PRESENT start: 1206210 length 51
+ Stream: column 23 section DATA start: 1206261 length 782
+ Stream: column 24 section PRESENT start: 1207043 length 54
+ Stream: column 24 section DATA start: 1207097 length 783
+ Stream: column 25 section PRESENT start: 1207880 length 51
+ Stream: column 25 section DATA start: 1207931 length 5448
+ Stream: column 26 section PRESENT start: 1213379 length 53
+ Stream: column 26 section DATA start: 1213432 length 12144
+ Stream: column 27 section PRESENT start: 1225576 length 53
+ Stream: column 27 section DATA start: 1225629 length 24618
+ Stream: column 28 section PRESENT start: 1250247 length 52
+ Stream: column 28 section DATA start: 1250299 length 24681
+ Stream: column 29 section PRESENT start: 1274980 length 51
+ Stream: column 29 section DATA start: 1275031 length 9927
+ Stream: column 30 section PRESENT start: 1284958 length 53
+ Stream: column 30 section DATA start: 1285011 length 19755
+ Stream: column 31 section PRESENT start: 1304766 length 51
+ Stream: column 31 section DATA start: 1304817 length 11009
+ Stream: column 31 section LENGTH start: 1315826 length 3722
+ Stream: column 31 section DICTIONARY_DATA start: 1319548 length 65435
+ Stream: column 32 section PRESENT start: 1384983 length 54
+ Stream: column 32 section DATA start: 1385037 length 11006
+ Stream: column 32 section LENGTH start: 1396043 length 3739
+ Stream: column 32 section DICTIONARY_DATA start: 1399782 length 66022
+ Stream: column 33 section PRESENT start: 1465804 length 52
+ Stream: column 33 section DATA start: 1465856 length 8045
+ Stream: column 33 section SECONDARY start: 1473901 length 9555
+ Stream: column 34 section PRESENT start: 1483456 length 58
+ Stream: column 34 section DATA start: 1483514 length 8082
+ Stream: column 34 section SECONDARY start: 1491596 length 9590
+ Stream: column 35 section PRESENT start: 1501186 length 51
+ Stream: column 35 section DATA start: 1501237 length 782
+ Stream: column 36 section PRESENT start: 1502019 length 54
+ Stream: column 36 section DATA start: 1502073 length 783
+ Encoding column 0: DIRECT
+ Encoding column 1: DIRECT
+ Encoding column 2: DIRECT_V2
+ Encoding column 3: DIRECT_V2
+ Encoding column 4: DIRECT_V2
+ Encoding column 5: DIRECT
+ Encoding column 6: DIRECT
+ Encoding column 7: DIRECT_V2
+ Encoding column 8: DIRECT_V2
+ Encoding column 9: DIRECT_V2
+ Encoding column 10: DIRECT_V2
+ Encoding column 11: DIRECT
+ Encoding column 12: DIRECT
+ Encoding column 13: DIRECT
+ Encoding column 14: DIRECT_V2
+ Encoding column 15: DIRECT_V2
+ Encoding column 16: DIRECT_V2
+ Encoding column 17: DIRECT
+ Encoding column 18: DIRECT
+ Encoding column 19: DICTIONARY_V2[6083]
+ Encoding column 20: DICTIONARY_V2[6081]
+ Encoding column 21: DIRECT_V2
+ Encoding column 22: DIRECT_V2
+ Encoding column 23: DIRECT
+ Encoding column 24: DIRECT
+ Encoding column 25: DIRECT
+ Encoding column 26: DIRECT_V2
+ Encoding column 27: DIRECT_V2
+ Encoding column 28: DIRECT_V2
+ Encoding column 29: DIRECT
+ Encoding column 30: DIRECT
+ Encoding column 31: DICTIONARY_V2[6083]
+ Encoding column 32: DICTIONARY_V2[6081]
+ Encoding column 33: DIRECT_V2
+ Encoding column 34: DIRECT_V2
+ Encoding column 35: DIRECT
+ Encoding column 36: DIRECT
+ Row group indices for column 1:
+ Entry 0: count: 7909 hasNull: true min: -64 max: 62 sum: -50203 positions: 0,0,0,0,0,0,0
+ Entry 1: count: 1264 hasNull: true min: -64 max: 62 sum: 10347 positions: 0,182,99,0,0,5937,2
+ Stripe: offset: 1503357 data: 1500017 rows: 12288 tail: 501 index: 2836
+ Stream: column 0 section ROW_INDEX start: 1503357 length 21
+ Stream: column 1 section ROW_INDEX start: 1503378 length 53
+ Stream: column 2 section ROW_INDEX start: 1503431 length 67
+ Stream: column 3 section ROW_INDEX start: 1503498 length 81
+ Stream: column 4 section ROW_INDEX start: 1503579 length 83
+ Stream: column 5 section ROW_INDEX start: 1503662 length 77
+ Stream: column 6 section ROW_INDEX start: 1503739 length 77
+ Stream: column 7 section ROW_INDEX start: 1503816 length 176
+ Stream: column 8 section ROW_INDEX start: 1503992 length 267
+ Stream: column 9 section ROW_INDEX start: 1504259 length 63
+ Stream: column 10 section ROW_INDEX start: 1504322 length 57
+ Stream: column 11 section ROW_INDEX start: 1504379 length 47
+ Stream: column 12 section ROW_INDEX start: 1504426 length 47
+ Stream: column 13 section ROW_INDEX start: 1504473 length 53
+ Stream: column 14 section ROW_INDEX start: 1504526 length 67
+ Stream: column 15 section ROW_INDEX start: 1504593 length 81
+ Stream: column 16 section ROW_INDEX start: 1504674 length 83
+ Stream: column 17 section ROW_INDEX start: 1504757 length 77
+ Stream: column 18 section ROW_INDEX start: 1504834 length 77
+ Stream: column 19 section ROW_INDEX start: 1504911 length 115
+ Stream: column 20 section ROW_INDEX start: 1505026 length 93
+ Stream: column 21 section ROW_INDEX start: 1505119 length 63
+ Stream: column 22 section ROW_INDEX start: 1505182 length 57
+ Stream: column 23 section ROW_INDEX start: 1505239 length 47
+ Stream: column 24 section ROW_INDEX start: 1505286 length 47
+ Stream: column 25 section ROW_INDEX start: 1505333 length 53
+ Stream: column 26 section ROW_INDEX start: 1505386 length 67
+ Stream: column 27 section ROW_INDEX start: 1505453 length 81
+ Stream: column 28 section ROW_INDEX start: 1505534 length 83
+ Stream: column 29 section ROW_INDEX start: 1505617 length 77
+ Stream: column 30 section ROW_INDEX start: 1505694 length 77
+ Stream: column 31 section ROW_INDEX start: 1505771 length 115
+ Stream: column 32 section ROW_INDEX start: 1505886 length 93
+ Stream: column 33 section ROW_INDEX start: 1505979 length 63
+ Stream: column 34 section ROW_INDEX start: 1506042 length 57
+ Stream: column 35 section ROW_INDEX start: 1506099 length 47
+ Stream: column 36 section ROW_INDEX start: 1506146 length 47
+ Stream: column 1 section PRESENT start: 1506193 length 51
+ Stream: column 1 section DATA start: 1506244 length 5448
+ Stream: column 2 section PRESENT start: 1511692 length 53
+ Stream: column 2 section DATA start: 1511745 length 12144
+ Stream: column 3 section PRESENT start: 1523889 length 53
+ Stream: column 3 section DATA start: 1523942 length 24618
+ Stream: column 4 section PRESENT start: 1548560 length 52
+ Stream: column 4 section DATA start: 1548612 length 24681
+ Stream: column 5 section PRESENT start: 1573293 length 51
+ Stream: column 5 section DATA start: 1573344 length 9927
+ Stream: column 6 section PRESENT start: 1583271 length 53
+ Stream: column 6 section DATA start: 1583324 length 19755
+ Stream: column 7 section DATA start: 1603079 length 258570
+ Stream: column 7 section LENGTH start: 1861649 length 108
+ Stream: column 8 section DATA start: 1861757 length 517341
+ Stream: column 8 section LENGTH start: 2379098 length 108
+ Stream: column 9 section PRESENT start: 2379206 length 52
+ Stream: column 9 section DATA start: 2379258 length 8045
+ Stream: column 9 section SECONDARY start: 2387303 length 9555
+ Stream: column 10 section PRESENT start: 2396858 length 58
+ Stream: column 10 section DATA start: 2396916 length 8082
+ Stream: column 10 section SECONDARY start: 2404998 length 9590
+ Stream: column 11 section PRESENT start: 2414588 length 51
+ Stream: column 11 section DATA start: 2414639 length 782
+ Stream: column 12 section PRESENT start: 2415421 length 54
+ Stream: column 12 section DATA start: 2415475 length 783
+ Stream: column 13 section PRESENT start: 2416258 length 51
+ Stream: column 13 section DATA start: 2416309 length 5448
+ Stream: column 14 section PRESENT start: 2421757 length 53
+ Stream: column 14 section DATA start: 2421810 length 12144
+ Stream: column 15 section PRESENT start: 2433954 length 53
+ Stream: column 15 section DATA start: 2434007 length 24618
+ Stream: column 16 section PRESENT start: 2458625 length 52
+ Stream: column 16 section DATA start: 2458677 length 24681
+ Stream: column 17 section PRESENT start: 2483358 length 51
+ Stream: column 17 section DATA start: 2483409 length 9927
+ Stream: column 18 section PRESENT start: 2493336 length 53
+ Stream: column 18 section DATA start: 2493389 length 19755
+ Stream: column 19 section PRESENT start: 2513144 length 51
+ Stream: column 19 section DATA start: 2513195 length 11009
+ Stream: column 19 section LENGTH start: 2524204 length 3722
+ Stream: column 19 section DICTIONARY_DATA start: 2527926 length 65435
+ Stream: column 20 section PRESENT start: 2593361 length 54
+ Stream: column 20 section DATA start: 2593415 length 11006
+ Stream: column 20 section LENGTH start: 2604421 length 3739
+ Stream: column 20 section DICTIONARY_DATA start: 2608160 length 66022
+ Stream: column 21 section PRESENT start: 2674182 length 52
+ Stream: column 21 section DATA start: 2674234 length 8045
+ Stream: column 21 section SECONDARY start: 2682279 length 9555
+ Stream: column 22 section PRESENT start: 2691834 length 58
+ Stream: column 22 section DATA start: 2691892 length 8082
+ Stream: column 22 section SECONDARY start: 2699974 length 9590
+ Stream: column 23 section PRESENT start: 2709564 length 51
+ Stream: column 23 section DATA start: 2709615 length 782
+ Stream: column 24 section PRESENT start: 2710397 length 54
+ Stream: column 24 section DATA start: 2710451 length 783
+ Stream: column 25 section PRESENT start: 2711234 length 51
+ Stream: column 25 section DATA start: 2711285 length 5448
+ Stream: column 26 section PRESENT start: 2716733 length 53
+ Stream: column 26 section DATA start: 2716786 length 12144
+ Stream: column 27 section PRESENT start: 2728930 length 53
+ Stream: column 27 section DATA start: 2728983 length 24618
+ Stream: column 28 section PRESENT start: 2753601 length 52
+ Stream: column 28 section DATA start: 2753653 length 24681
+ Stream: column 29 section PRESENT start: 2778334 length 51
+ Stream: column 29 section DATA start: 2778385 length 9927
+ Stream: column 30 section PRESENT start: 2788312 length 53
+ Stream: column 30 section DATA start: 2788365 length 19755
+ Stream: column 31 section PRESENT start: 2808120 length 51
+ Stream: column 31 section DATA start: 2808171 length 11009
+ Stream: column 31 section LENGTH start: 2819180 length 3722
+ Stream: column 31 section DICTIONARY_DATA start: 2822902 length 65435
+ Stream: column 32 section PRESENT start: 2888337 length 54
+ Stream: column 32 section DATA start: 2888391 length 11006
+ Stream: column 32 section LENGTH start: 2899397 length 3739
+ Stream: column 32 section DICTIONARY_DATA start: 2903136 length 66022
+ Stream: column 33 section PRESENT start: 2969158 length 52
+ Stream: column 33 section DATA start: 2969210 length 8045
+ Stream: column 33 section SECONDARY start: 2977255 length 9555
+ Stream: column 34 section PRESENT start: 2986810 length 58
+ Stream: column 34 section DATA start: 2986868 length 8082
+ Stream: column 34 section SECONDARY start: 2994950 length 9590
+ Stream: column 35 section PRESENT start: 3004540 length 51
+ Stream: column 35 section DATA start: 3004591 length 782
+ Stream: column 36 section PRESENT start: 3005373 length 54
+ Stream: column 36 section DATA start: 3005427 length 783
+ Encoding column 0: DIRECT
+ Encoding column 1: DIRECT
+ Encoding column 2: DIRECT_V2
+ Encoding column 3: DIRECT_V2
+ Encoding column 4: DIRECT_V2
+ Encoding column 5: DIRECT
+ Encoding column 6: DIRECT
+ Encoding column 7: DIRECT_V2
+ Encoding column 8: DIRECT_V2
+ Encoding column 9: DIRECT_V2
+ Encoding column 10: DIRECT_V2
+ Encoding column 11: DIRECT
+ Encoding column 12: DIRECT
+ Encoding column 13: DIRECT
+ Encoding column 14: DIRECT_V2
+ Encoding column 15: DIRECT_V2
+ Encoding column 16: DIRECT_V2
+ Encoding column 17: DIRECT
+ Encoding column 18: DIRECT
+ Encoding column 19: DICTIONARY_V2[6083]
+ Encoding column 20: DICTIONARY_V2[6081]
+ Encoding column 21: DIRECT_V2
+ Encoding column 22: DIRECT_V2
+ Encoding column 23: DIRECT
+ Encoding column 24: DIRECT
+ Encoding column 25: DIRECT
+ Encoding column 26: DIRECT_V2
+ Encoding column 27: DIRECT_V2
+ Encoding column 28: DIRECT_V2
+ Encoding column 29: DIRECT
+ Encoding column 30: DIRECT
+ Encoding column 31: DICTIONARY_V2[6083]
+ Encoding column 32: DICTIONARY_V2[6081]
+ Encoding column 33: DIRECT_V2
+ Encoding column 34: DIRECT_V2
+ Encoding column 35: DIRECT
+ Encoding column 36: DIRECT
+ Row group indices for column 1:
+ Entry 0: count: 7909 hasNull: true min: -64 max: 62 sum: -50203 positions: 0,0,0,0,0,0,0
+ Entry 1: count: 1264 hasNull: true min: -64 max: 62 sum: 10347 positions: 0,182,99,0,0,5937,2
+
+File length: 3007981 bytes
+Padding length: 0 bytes
+Padding ratio: 0%
+________________________________________________________________________________________________________________________
+
+-- END ORC FILE DUMP --
+-50 -13326 528534767 NULL -50.0 -13326.0 18f2de7d-0c69-4052-9c4b-64f196d6d589 9edba9e8-1f91-47e7-b31f-451eeb5feb7781ba3a3e-c22e-4412-91a2-86ad05ae1ca8 1969-12-31 15:59:46.674 1969-12-31 16:00:07.875 true NULL -50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:07.875 true NULL -50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:07.875 true NULL
[13/50] [abbrv] hive git commit: HIVE-13307: LLAP: Slider package
should contain permanent functions (addendum)
Posted by jd...@apache.org.
HIVE-13307: LLAP: Slider package should contain permanent functions (addendum)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b1c45029
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b1c45029
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b1c45029
Branch: refs/heads/llap
Commit: b1c45029ed3652eda9db6650da38ba653d4ada93
Parents: 4fabd03
Author: Gopal V <go...@apache.org>
Authored: Fri Mar 25 00:18:44 2016 -0700
Committer: Gopal V <go...@apache.org>
Committed: Fri Mar 25 00:19:35 2016 -0700
----------------------------------------------------------------------
ql/src/test/queries/clientpositive/llap_udf.q | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b1c45029/ql/src/test/queries/clientpositive/llap_udf.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/llap_udf.q b/ql/src/test/queries/clientpositive/llap_udf.q
index c964f2b..2224bd5 100644
--- a/ql/src/test/queries/clientpositive/llap_udf.q
+++ b/ql/src/test/queries/clientpositive/llap_udf.q
@@ -3,7 +3,7 @@ set hive.explain.user=false;
set hive.execution.mode=llap;
set hive.llap.execution.mode=all;
set hive.fetch.task.conversion=none;
-set hive.llap.daemon.allow.permanent.fns=true;
+set hive.llap.allow.permanent.fns=true;
drop table if exists src_orc;
create table src_orc stored as orc as select * from src;
@@ -37,11 +37,11 @@ DROP FUNCTION test_udf4;
EXPLAIN
SELECT test_udf0(cast(key as string)) from src_orc;
-set hive.llap.daemon.allow.permanent.fns=false;
+set hive.llap.allow.permanent.fns=false;
EXPLAIN
SELECT test_udf3(cast(key as string)) from src_orc;
drop table if exists src_orc;
-set hive.execution.mode=container;
\ No newline at end of file
+set hive.execution.mode=container;
[34/50] [abbrv] hive git commit: HIVE-13111: Fix timestamp /
interval_day_time wrong results with HIVE-9862 (Matt McCline,
reviewed by Jason Dere)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticTimestampColumn.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticTimestampColumn.txt b/ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticTimestampColumn.txt
index 6241ee2..63cebaf 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticTimestampColumn.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticTimestampColumn.txt
@@ -18,28 +18,155 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
+import java.sql.Timestamp;
+
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil;
import org.apache.hadoop.hive.ql.exec.vector.*;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
import org.apache.hadoop.hive.ql.util.DateTimeMath;
+import org.apache.hadoop.hive.serde2.io.DateWritable;
/**
- * Generated from template DateColumnArithmeticTimestampColumn.txt, which covers binary arithmetic
- * expressions between a date column and a timestamp column.
+ * Generated from template DateColumnArithmeticTimestampColumn.txt, a class
+ * which covers binary arithmetic expressions between a date column and timestamp column.
*/
-public class <ClassName> extends <BaseClassName> {
+public class <ClassName> extends VectorExpression {
private static final long serialVersionUID = 1L;
+ private int colNum1;
+ private int colNum2;
+ private int outputColumn;
+ private Timestamp scratchTimestamp1;
+ private DateTimeMath dtm = new DateTimeMath();
+
public <ClassName>(int colNum1, int colNum2, int outputColumn) {
- super(colNum1, colNum2, outputColumn);
+ this.colNum1 = colNum1;
+ this.colNum2 = colNum2;
+ this.outputColumn = outputColumn;
+ scratchTimestamp1 = new Timestamp(0);
}
public <ClassName>() {
- super();
+ }
+
+ @Override
+ public void evaluate(VectorizedRowBatch batch) {
+
+ if (childExpressions != null) {
+ super.evaluateChildren(batch);
+ }
+
+ // Input #1 is type Date (days). For the math we convert it to a timestamp.
+ LongColumnVector inputColVector1 = (LongColumnVector) batch.cols[colNum1];
+
+ // Input #2 is type <OperandType2>.
+ <InputColumnVectorType2> inputColVector2 = (<InputColumnVectorType2>) batch.cols[colNum2];
+
+ // Output is type <ReturnType>.
+ <OutputColumnVectorType> outputColVector = (<OutputColumnVectorType>) batch.cols[outputColumn];
+
+ int[] sel = batch.selected;
+ int n = batch.size;
+ long[] vector1 = inputColVector1.vector;
+
+ // return immediately if batch is empty
+ if (n == 0) {
+ return;
+ }
+
+ outputColVector.isRepeating =
+ inputColVector1.isRepeating && inputColVector2.isRepeating
+ || inputColVector1.isRepeating && !inputColVector1.noNulls && inputColVector1.isNull[0]
+ || inputColVector2.isRepeating && !inputColVector2.noNulls && inputColVector2.isNull[0];
+
+ // Handle nulls first
+ NullUtil.propagateNullsColCol(
+ inputColVector1, inputColVector2, outputColVector, sel, n, batch.selectedInUse);
+
+ /* Disregard nulls for processing. In other words,
+ * the arithmetic operation is performed even if one or
+ * more inputs are null. This is to improve speed by avoiding
+ * conditional checks in the inner loop.
+ */
+ if (inputColVector1.isRepeating && inputColVector2.isRepeating) {
+ scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[0]));
+ dtm.<OperatorMethod>(
+ scratchTimestamp1, inputColVector2.asScratch<CamelOperandType2>(0), outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(0);
+ } else if (inputColVector1.isRepeating) {
+ scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[0]));
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ dtm.<OperatorMethod>(
+ scratchTimestamp1, inputColVector2.asScratch<CamelOperandType2>(i), outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ dtm.<OperatorMethod>(
+ scratchTimestamp1, inputColVector2.asScratch<CamelOperandType2>(i), outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ }
+ }
+ } else if (inputColVector2.isRepeating) {
+ <HiveOperandType2> value2 = inputColVector2.asScratch<CamelOperandType2>(0);
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[i]));
+ dtm.<OperatorMethod>(
+ scratchTimestamp1, value2, outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[i]));
+ dtm.<OperatorMethod>(
+ scratchTimestamp1, value2, outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ }
+ }
+ } else {
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[i]));
+ dtm.<OperatorMethod>(
+ scratchTimestamp1, inputColVector2.asScratch<CamelOperandType2>(i), outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[i]));
+ dtm.<OperatorMethod>(
+ scratchTimestamp1, inputColVector2.asScratch<CamelOperandType2>(i), outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ }
+ }
+ }
+
+ /* For the case when the output can have null values, follow
+ * the convention that the data values must be 1 for long and
+ * NaN for double. This is to prevent possible later zero-divide errors
+ * in complex arithmetic expressions like col2 / (col1 - 1)
+ * in the case when some col1 entries are null.
+ */
+ NullUtil.setNullDataEntries<CamelReturnType>(outputColVector, batch.selectedInUse, sel, n);
+ }
+
+ @Override
+ public int getOutputColumn() {
+ return outputColumn;
+ }
+
+ @Override
+ public String getOutputType() {
+ return "<ReturnType>";
}
@Override
@@ -49,7 +176,7 @@ public class <ClassName> extends <BaseClassName> {
VectorExpressionDescriptor.Mode.PROJECTION)
.setNumArguments(2)
.setArgumentTypes(
- VectorExpressionDescriptor.ArgumentType.getType("<OperandType1>"),
+ VectorExpressionDescriptor.ArgumentType.getType("date"),
VectorExpressionDescriptor.ArgumentType.getType("<OperandType2>"))
.setInputExpressionTypes(
VectorExpressionDescriptor.InputExpressionType.COLUMN,
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticTimestampColumnBase.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticTimestampColumnBase.txt b/ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticTimestampColumnBase.txt
deleted file mode 100644
index a61b769..0000000
--- a/ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticTimestampColumnBase.txt
+++ /dev/null
@@ -1,171 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
-
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil;
-import org.apache.hadoop.hive.ql.exec.vector.*;
-import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
-import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
-import org.apache.hadoop.hive.serde2.io.DateWritable;
-
-/**
- * Generated from template DateColumnArithmeticTimestampColumnBase.txt, a base class
- * which covers binary arithmetic expressions between a date column and timestamp column.
- */
-public abstract class <BaseClassName> extends VectorExpression {
-
- private static final long serialVersionUID = 1L;
-
- private int colNum1;
- private int colNum2;
- private int outputColumn;
- private PisaTimestamp scratchPisaTimestamp;
-
- public <BaseClassName>(int colNum1, int colNum2, int outputColumn) {
- this.colNum1 = colNum1;
- this.colNum2 = colNum2;
- this.outputColumn = outputColumn;
- scratchPisaTimestamp = new PisaTimestamp();
- }
-
- public <BaseClassName>() {
- }
-
- @Override
- public void evaluate(VectorizedRowBatch batch) {
-
- if (childExpressions != null) {
- super.evaluateChildren(batch);
- }
-
- // Input #1 is type Date (epochDays).
- LongColumnVector inputColVector1 = (LongColumnVector) batch.cols[colNum1];
-
- // Input #2 is type timestamp/interval_day_time.
- TimestampColumnVector inputColVector2 = (TimestampColumnVector) batch.cols[colNum2];
-
- // Output is type timestamp.
- TimestampColumnVector outputColVector = (TimestampColumnVector) batch.cols[outputColumn];
-
- int[] sel = batch.selected;
- int n = batch.size;
- long[] vector1 = inputColVector1.vector;
-
- // return immediately if batch is empty
- if (n == 0) {
- return;
- }
-
- outputColVector.isRepeating =
- inputColVector1.isRepeating && inputColVector2.isRepeating
- || inputColVector1.isRepeating && !inputColVector1.noNulls && inputColVector1.isNull[0]
- || inputColVector2.isRepeating && !inputColVector2.noNulls && inputColVector2.isNull[0];
-
- // Handle nulls first
- NullUtil.propagateNullsColCol(
- inputColVector1, inputColVector2, outputColVector, sel, n, batch.selectedInUse);
-
- /* Disregard nulls for processing. In other words,
- * the arithmetic operation is performed even if one or
- * more inputs are null. This is to improve speed by avoiding
- * conditional checks in the inner loop.
- */
- if (inputColVector1.isRepeating && inputColVector2.isRepeating) {
- outputColVector.<OperatorMethod>(
- scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector1[0])),
- inputColVector2.asScratchPisaTimestamp(0),
- 0);
- } else if (inputColVector1.isRepeating) {
- PisaTimestamp value1 =
- scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector1[0]));
- if (batch.selectedInUse) {
- for(int j = 0; j != n; j++) {
- int i = sel[j];
- outputColVector.<OperatorMethod>(
- value1,
- inputColVector2.asScratchPisaTimestamp(i),
- i);
- }
- } else {
- for(int i = 0; i != n; i++) {
- outputColVector.<OperatorMethod>(
- value1,
- inputColVector2.asScratchPisaTimestamp(i),
- i);
- }
- }
- } else if (inputColVector2.isRepeating) {
- PisaTimestamp value2 = inputColVector2.asScratchPisaTimestamp(0);
- if (batch.selectedInUse) {
- for(int j = 0; j != n; j++) {
- int i = sel[j];
- outputColVector.<OperatorMethod>(
- scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector1[i])),
- value2,
- i);
- }
- } else {
- for(int i = 0; i != n; i++) {
- outputColVector.<OperatorMethod>(
- scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector1[i])),
- value2,
- i);
- }
- }
- } else {
- if (batch.selectedInUse) {
- for(int j = 0; j != n; j++) {
- int i = sel[j];
- outputColVector.<OperatorMethod>(
- scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector1[i])),
- inputColVector2.asScratchPisaTimestamp(i),
- i);
- }
- } else {
- for(int i = 0; i != n; i++) {
- outputColVector.<OperatorMethod>(
- scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector1[i])),
- inputColVector2.asScratchPisaTimestamp(i),
- i);
- }
- }
- }
-
- /* For the case when the output can have null values, follow
- * the convention that the data values must be 1 for long and
- * NaN for double. This is to prevent possible later zero-divide errors
- * in complex arithmetic expressions like col2 / (col1 - 1)
- * in the case when some col1 entries are null.
- */
- NullUtil.setNullDataEntriesTimestamp(outputColVector, batch.selectedInUse, sel, n);
- }
-
- @Override
- public int getOutputColumn() {
- return outputColumn;
- }
-
- @Override
- public String getOutputType() {
- return "timestamp";
- }
-}
-
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticTimestampScalar.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticTimestampScalar.txt b/ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticTimestampScalar.txt
index b813d11..7aee529 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticTimestampScalar.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticTimestampScalar.txt
@@ -19,32 +19,123 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
import java.sql.Timestamp;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
-import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
-import org.apache.hive.common.util.DateUtils;
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
-import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
import org.apache.hadoop.hive.ql.exec.vector.*;
import org.apache.hadoop.hive.ql.util.DateTimeMath;
+import org.apache.hadoop.hive.serde2.io.DateWritable;
/**
- * Generated from template DateColumnArithmeticTimestampScalar.txt, which covers binary arithmetic
- * expressions between a date column and a timestamp scalar.
+ * Generated from template DateColumnArithmeticTimestampScalarBase.txt, a base class
+ * which covers binary arithmetic expressions between a date column and a timestamp scalar.
*/
-public class <ClassName> extends <BaseClassName> {
+public class <ClassName> extends VectorExpression {
private static final long serialVersionUID = 1L;
- public <ClassName>(int colNum, <ScalarHiveTimestampType2> value, int outputColumn) {
- super(colNum, <PisaTimestampConversion2>, outputColumn);
+ private int colNum;
+ private <HiveOperandType2> value;
+ private int outputColumn;
+ private Timestamp scratchTimestamp1;
+ private DateTimeMath dtm = new DateTimeMath();
+
+ public <ClassName>(int colNum, <HiveOperandType2> value, int outputColumn) {
+ this.colNum = colNum;
+ this.value = value;
+ this.outputColumn = outputColumn;
+ scratchTimestamp1 = new Timestamp(0);
}
public <ClassName>() {
- super();
+ }
+
+ @Override
+ public void evaluate(VectorizedRowBatch batch) {
+
+ if (childExpressions != null) {
+ super.evaluateChildren(batch);
+ }
+
+ // Input #1 is type date (days). For the math we convert it to a timestamp.
+ LongColumnVector inputColVector1 = (LongColumnVector) batch.cols[colNum];
+
+ // Output is type <ReturnType>.
+ <OutputColumnVectorType> outputColVector = (<OutputColumnVectorType>) batch.cols[outputColumn];
+
+ int[] sel = batch.selected;
+ boolean[] inputIsNull = inputColVector1.isNull;
+ boolean[] outputIsNull = outputColVector.isNull;
+ outputColVector.noNulls = inputColVector1.noNulls;
+ outputColVector.isRepeating = inputColVector1.isRepeating;
+ int n = batch.size;
+ long[] vector1 = inputColVector1.vector;
+
+ // return immediately if batch is empty
+ if (n == 0) {
+ return;
+ }
+
+ if (inputColVector1.isRepeating) {
+ scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[0]));
+ dtm.<OperatorMethod>(
+ scratchTimestamp1, value, outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(0);
+ // Even if there are no nulls, we always copy over entry 0. Simplifies code.
+ outputIsNull[0] = inputIsNull[0];
+ } else if (inputColVector1.noNulls) {
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[i]));
+ dtm.<OperatorMethod>(
+ scratchTimestamp1, value, outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[i]));
+ dtm.<OperatorMethod>(
+ scratchTimestamp1, value, outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ }
+ }
+ } else /* there are nulls */ {
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[i]));
+ dtm.<OperatorMethod>(
+ scratchTimestamp1, value, outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ outputIsNull[i] = inputIsNull[i];
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[i]));
+ dtm.<OperatorMethod>(
+ scratchTimestamp1, value, outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ }
+ System.arraycopy(inputIsNull, 0, outputIsNull, 0, n);
+ }
+ }
+
+ NullUtil.setNullOutputEntriesColScalar(outputColVector, batch.selectedInUse, sel, n);
+ }
+
+ @Override
+ public int getOutputColumn() {
+ return outputColumn;
+ }
+
+ @Override
+ public String getOutputType() {
+ return "<ReturnType>";
}
@Override
@@ -54,7 +145,7 @@ public class <ClassName> extends <BaseClassName> {
VectorExpressionDescriptor.Mode.PROJECTION)
.setNumArguments(2)
.setArgumentTypes(
- VectorExpressionDescriptor.ArgumentType.getType("<OperandType1>"),
+ VectorExpressionDescriptor.ArgumentType.getType("date"),
VectorExpressionDescriptor.ArgumentType.getType("<OperandType2>"))
.setInputExpressionTypes(
VectorExpressionDescriptor.InputExpressionType.COLUMN,
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticTimestampScalarBase.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticTimestampScalarBase.txt b/ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticTimestampScalarBase.txt
deleted file mode 100644
index d64fba0..0000000
--- a/ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticTimestampScalarBase.txt
+++ /dev/null
@@ -1,137 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
-
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
-import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil;
-import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
-import org.apache.hadoop.hive.ql.exec.vector.*;
-import org.apache.hadoop.hive.serde2.io.DateWritable;
-
-/**
- * Generated from template DateColumnArithmeticTimestampScalarBase.txt, a base class
- * which covers binary arithmetic expressions between a date column and a timestamp scalar.
- */
-public abstract class <BaseClassName> extends VectorExpression {
-
- private static final long serialVersionUID = 1L;
-
- private int colNum;
- private PisaTimestamp value;
- private int outputColumn;
- private PisaTimestamp scratchPisaTimestamp;
-
- public <BaseClassName>(int colNum, PisaTimestamp value, int outputColumn) {
- this.colNum = colNum;
- this.value = value;
- this.outputColumn = outputColumn;
- scratchPisaTimestamp = new PisaTimestamp();
- }
-
- public <BaseClassName>() {
- }
-
- @Override
- public void evaluate(VectorizedRowBatch batch) {
-
- if (childExpressions != null) {
- super.evaluateChildren(batch);
- }
-
- // Input #1 is type date (epochDays).
- LongColumnVector inputColVector1 = (LongColumnVector) batch.cols[colNum];
-
- // Output is type timestamp.
- TimestampColumnVector outputColVector = (TimestampColumnVector) batch.cols[outputColumn];
-
- int[] sel = batch.selected;
- boolean[] inputIsNull = inputColVector1.isNull;
- boolean[] outputIsNull = outputColVector.isNull;
- outputColVector.noNulls = inputColVector1.noNulls;
- outputColVector.isRepeating = inputColVector1.isRepeating;
- int n = batch.size;
- long[] vector1 = inputColVector1.vector;
-
- // return immediately if batch is empty
- if (n == 0) {
- return;
- }
-
- if (inputColVector1.isRepeating) {
- outputColVector.<OperatorMethod>(
- scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector1[0])),
- value,
- 0);
-
- // Even if there are no nulls, we always copy over entry 0. Simplifies code.
- outputIsNull[0] = inputIsNull[0];
- } else if (inputColVector1.noNulls) {
- if (batch.selectedInUse) {
- for(int j = 0; j != n; j++) {
- int i = sel[j];
- outputColVector.<OperatorMethod>(
- scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector1[i])),
- value,
- i);
- }
- } else {
- for(int i = 0; i != n; i++) {
- outputColVector.<OperatorMethod>(
- scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector1[i])),
- value,
- i);
- }
- }
- } else /* there are nulls */ {
- if (batch.selectedInUse) {
- for(int j = 0; j != n; j++) {
- int i = sel[j];
- outputColVector.<OperatorMethod>(
- scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector1[i])),
- value,
- i);
- outputIsNull[i] = inputIsNull[i];
- }
- } else {
- for(int i = 0; i != n; i++) {
- outputColVector.<OperatorMethod>(
- scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector1[i])),
- value,
- i);
- }
- System.arraycopy(inputIsNull, 0, outputIsNull, 0, n);
- }
- }
-
- NullUtil.setNullOutputEntriesColScalar(outputColVector, batch.selectedInUse, sel, n);
- }
-
- @Override
- public int getOutputColumn() {
- return outputColumn;
- }
-
- @Override
- public String getOutputType() {
- return "timestamp";
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/DateScalarArithmeticIntervalYearMonthColumn.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/DateScalarArithmeticIntervalYearMonthColumn.txt b/ql/src/gen/vectorization/ExpressionTemplates/DateScalarArithmeticIntervalYearMonthColumn.txt
index 653565e..c68ac34 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/DateScalarArithmeticIntervalYearMonthColumn.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/DateScalarArithmeticIntervalYearMonthColumn.txt
@@ -18,6 +18,8 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
+import java.sql.Date;
+import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
import org.apache.hadoop.hive.ql.exec.vector.*;
@@ -33,6 +35,7 @@ import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil;
import org.apache.hadoop.hive.ql.util.DateTimeMath;
+import org.apache.hadoop.hive.serde2.io.DateWritable;
/**
* Generated from template DateTimeScalarArithmeticIntervalYearMonthColumn.txt.
@@ -44,14 +47,18 @@ public class <ClassName> extends VectorExpression {
private static final long serialVersionUID = 1L;
private int colNum;
- private long value;
+ private Date value;
private int outputColumn;
+ private HiveIntervalYearMonth scratchIntervalYearMonth2;
+ private Date outputDate;
private DateTimeMath dtm = new DateTimeMath();
public <ClassName>(long value, int colNum, int outputColumn) {
this.colNum = colNum;
- this.value = value;
+ this.value = new Date(DateWritable.daysToMillis((int) value));
this.outputColumn = outputColumn;
+ scratchIntervalYearMonth2 = new HiveIntervalYearMonth();
+ outputDate = new Date(0);
}
public <ClassName>() {
@@ -70,18 +77,18 @@ public class <ClassName> extends VectorExpression {
}
// Input #2 is type Interval_Year_Month (months).
- LongColumnVector inputColVector = (LongColumnVector) batch.cols[colNum];
+ LongColumnVector inputColVector2 = (LongColumnVector) batch.cols[colNum];
// Output is type Date.
LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumn];
int[] sel = batch.selected;
- boolean[] inputIsNull = inputColVector.isNull;
+ boolean[] inputIsNull = inputColVector2.isNull;
boolean[] outputIsNull = outputColVector.isNull;
- outputColVector.noNulls = inputColVector.noNulls;
- outputColVector.isRepeating = inputColVector.isRepeating;
+ outputColVector.noNulls = inputColVector2.noNulls;
+ outputColVector.isRepeating = inputColVector2.isRepeating;
int n = batch.size;
- long[] vector = inputColVector.vector;
+ long[] vector2 = inputColVector2.vector;
long[] outputVector = outputColVector.vector;
// return immediately if batch is empty
@@ -89,32 +96,46 @@ public class <ClassName> extends VectorExpression {
return;
}
- if (inputColVector.isRepeating) {
- outputVector[0] = dtm.addMonthsToDays(value, <OperatorSymbol> (int) vector[0]);
-
- // Even if there are no nulls, we always copy over entry 0. Simplifies code.
+ if (inputColVector2.isRepeating) {
+ scratchIntervalYearMonth2.set((int) vector2[0]);
+ dtm.<OperatorMethod>(
+ value, scratchIntervalYearMonth2, outputDate);
+ outputVector[0] = DateWritable.dateToDays(outputDate);
+ // Even if there are no nulls, we always copy over entry 0. Simplifies code.
outputIsNull[0] = inputIsNull[0];
- } else if (inputColVector.noNulls) {
+ } else if (inputColVector2.noNulls) {
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputVector[i] = dtm.addMonthsToDays(value, <OperatorSymbol> (int) vector[i]);
+ scratchIntervalYearMonth2.set((int) vector2[i]);
+ dtm.<OperatorMethod>(
+ value, scratchIntervalYearMonth2, outputDate);
+ outputVector[i] = DateWritable.dateToDays(outputDate);
}
} else {
for(int i = 0; i != n; i++) {
- outputVector[i] = dtm.addMonthsToDays(value, <OperatorSymbol> (int) vector[i]);
+ scratchIntervalYearMonth2.set((int) vector2[i]);
+ dtm.<OperatorMethod>(
+ value, scratchIntervalYearMonth2, outputDate);
+ outputVector[i] = DateWritable.dateToDays(outputDate);
}
}
} else { /* there are nulls */
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputVector[i] = dtm.addMonthsToDays(value, <OperatorSymbol> (int) vector[i]);
+ scratchIntervalYearMonth2.set((int) vector2[i]);
+ dtm.<OperatorMethod>(
+ value, scratchIntervalYearMonth2, outputDate);
+ outputVector[i] = DateWritable.dateToDays(outputDate);
outputIsNull[i] = inputIsNull[i];
}
} else {
for(int i = 0; i != n; i++) {
- outputVector[i] = dtm.addMonthsToDays(value, <OperatorSymbol> (int) vector[i]);
+ scratchIntervalYearMonth2.set((int) vector2[i]);
+ dtm.<OperatorMethod>(
+ value, scratchIntervalYearMonth2, outputDate);
+ outputVector[i] = DateWritable.dateToDays(outputDate);
}
System.arraycopy(inputIsNull, 0, outputIsNull, 0, n);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/DateScalarArithmeticTimestampColumn.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/DateScalarArithmeticTimestampColumn.txt b/ql/src/gen/vectorization/ExpressionTemplates/DateScalarArithmeticTimestampColumn.txt
index e93bed5..cb6b750 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/DateScalarArithmeticTimestampColumn.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/DateScalarArithmeticTimestampColumn.txt
@@ -18,45 +18,141 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
+import java.sql.Timestamp;
+
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
import org.apache.hadoop.hive.ql.exec.vector.*;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
/*
* Because of the templatized nature of the code, either or both
* of these ColumnVector imports may be needed. Listing both of them
* rather than using ....vectorization.*;
*/
-import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil;
import org.apache.hadoop.hive.ql.util.DateTimeMath;
+import org.apache.hadoop.hive.serde2.io.DateWritable;
/**
- * Generated from template DateScalarArithmeticTimestampColumn.txt.
+ * Generated from template DateTimeScalarArithmeticTimestampColumnBase.txt.
* Implements a vectorized arithmetic operator with a scalar on the left and a
* column vector on the right. The result is output to an output column vector.
*/
-public class <ClassName> extends <BaseClassName> {
+public class <ClassName> extends VectorExpression {
private static final long serialVersionUID = 1L;
+ private int colNum;
+ private Timestamp value;
+ private int outputColumn;
+ private DateTimeMath dtm = new DateTimeMath();
+
public <ClassName>(long value, int colNum, int outputColumn) {
- super(value, colNum, outputColumn);
+ this.colNum = colNum;
+ // Scalar input #1 is type date (days). For the math we convert it to a timestamp.
+ this.value = new Timestamp(0);
+ this.value.setTime(DateWritable.daysToMillis((int) value));
+ this.outputColumn = outputColumn;
}
public <ClassName>() {
}
@Override
+ /**
+ * Method to evaluate scalar-column operation in vectorized fashion.
+ *
+ * @batch a package of rows with each column stored in a vector
+ */
+ public void evaluate(VectorizedRowBatch batch) {
+
+ if (childExpressions != null) {
+ super.evaluateChildren(batch);
+ }
+
+ // Input #2 is type <OperandType2>.
+ <InputColumnVectorType2> inputColVector2 = (<InputColumnVectorType2>) batch.cols[colNum];
+
+ // Output is type <ReturnType>.
+ <OutputColumnVectorType> outputColVector = (<OutputColumnVectorType>) batch.cols[outputColumn];
+
+ int[] sel = batch.selected;
+ boolean[] inputIsNull = inputColVector2.isNull;
+ boolean[] outputIsNull = outputColVector.isNull;
+ outputColVector.noNulls = inputColVector2.noNulls;
+ outputColVector.isRepeating = inputColVector2.isRepeating;
+ int n = batch.size;
+
+ // return immediately if batch is empty
+ if (n == 0) {
+ return;
+ }
+
+ if (inputColVector2.isRepeating) {
+ dtm.<OperatorMethod>(
+ value, inputColVector2.asScratch<CamelOperandType2>(0), outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(0);
+ // Even if there are no nulls, we always copy over entry 0. Simplifies code.
+ outputIsNull[0] = inputIsNull[0];
+ } else if (inputColVector2.noNulls) {
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ dtm.<OperatorMethod>(
+ value, inputColVector2.asScratch<CamelOperandType2>(i), outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ dtm.<OperatorMethod>(
+ value, inputColVector2.asScratch<CamelOperandType2>(i), outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ }
+ }
+ } else { /* there are nulls */
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ dtm.<OperatorMethod>(
+ value, inputColVector2.asScratch<CamelOperandType2>(i), outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ outputIsNull[i] = inputIsNull[i];
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ dtm.<OperatorMethod>(
+ value, inputColVector2.asScratch<CamelOperandType2>(i), outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ }
+ System.arraycopy(inputIsNull, 0, outputIsNull, 0, n);
+ }
+ }
+
+ NullUtil.setNullOutputEntriesColScalar(outputColVector, batch.selectedInUse, sel, n);
+ }
+
+ @Override
+ public int getOutputColumn() {
+ return outputColumn;
+ }
+
+ @Override
+ public String getOutputType() {
+ return "<ReturnType>";
+ }
+
+ @Override
public VectorExpressionDescriptor.Descriptor getDescriptor() {
return (new VectorExpressionDescriptor.Builder())
.setMode(
VectorExpressionDescriptor.Mode.PROJECTION)
.setNumArguments(2)
.setArgumentTypes(
- VectorExpressionDescriptor.ArgumentType.getType("<OperandType1>"),
+ VectorExpressionDescriptor.ArgumentType.getType("date"),
VectorExpressionDescriptor.ArgumentType.getType("<OperandType2>"))
.setInputExpressionTypes(
VectorExpressionDescriptor.InputExpressionType.SCALAR,
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/DateScalarArithmeticTimestampColumnBase.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/DateScalarArithmeticTimestampColumnBase.txt b/ql/src/gen/vectorization/ExpressionTemplates/DateScalarArithmeticTimestampColumnBase.txt
deleted file mode 100644
index a1f4e6f..0000000
--- a/ql/src/gen/vectorization/ExpressionTemplates/DateScalarArithmeticTimestampColumnBase.txt
+++ /dev/null
@@ -1,147 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
-
-import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
-import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
-import org.apache.hadoop.hive.ql.exec.vector.*;
-
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
-/*
- * Because of the templatized nature of the code, either or both
- * of these ColumnVector imports may be needed. Listing both of them
- * rather than using ....vectorization.*;
- */
-import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil;
-import org.apache.hadoop.hive.serde2.io.DateWritable;
-
-/**
- * Generated from template DateTimeScalarArithmeticTimestampColumnBase.txt.
- * Implements a vectorized arithmetic operator with a scalar on the left and a
- * column vector on the right. The result is output to an output column vector.
- */
-public abstract class <BaseClassName> extends VectorExpression {
-
- private static final long serialVersionUID = 1L;
-
- private int colNum;
- private PisaTimestamp value;
- private int outputColumn;
-
- public <BaseClassName>(long value, int colNum, int outputColumn) {
- this.colNum = colNum;
- this.value = new PisaTimestamp().updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) value));
- this.outputColumn = outputColumn;
- }
-
- public <BaseClassName>() {
- }
-
- @Override
- /**
- * Method to evaluate scalar-column operation in vectorized fashion.
- *
- * @batch a package of rows with each column stored in a vector
- */
- public void evaluate(VectorizedRowBatch batch) {
-
- if (childExpressions != null) {
- super.evaluateChildren(batch);
- }
-
- // Input #2 is type timestamp/interval_day_time.
- TimestampColumnVector inputColVector2 = (TimestampColumnVector) batch.cols[colNum];
-
- // Output is type timestamp.
- TimestampColumnVector outputColVector = (TimestampColumnVector) batch.cols[outputColumn];
-
- int[] sel = batch.selected;
- boolean[] inputIsNull = inputColVector2.isNull;
- boolean[] outputIsNull = outputColVector.isNull;
- outputColVector.noNulls = inputColVector2.noNulls;
- outputColVector.isRepeating = inputColVector2.isRepeating;
- int n = batch.size;
-
- // return immediately if batch is empty
- if (n == 0) {
- return;
- }
-
- if (inputColVector2.isRepeating) {
- outputColVector.<OperatorMethod>(
- value,
- inputColVector2.asScratchPisaTimestamp(0),
- 0);
-
- // Even if there are no nulls, we always copy over entry 0. Simplifies code.
- outputIsNull[0] = inputIsNull[0];
- } else if (inputColVector2.noNulls) {
- if (batch.selectedInUse) {
- for(int j = 0; j != n; j++) {
- int i = sel[j];
- outputColVector.<OperatorMethod>(
- value,
- inputColVector2.asScratchPisaTimestamp(i),
- i);
- }
- } else {
- for(int i = 0; i != n; i++) {
- outputColVector.<OperatorMethod>(
- value,
- inputColVector2.asScratchPisaTimestamp(i),
- i);
- }
- }
- } else { /* there are nulls */
- if (batch.selectedInUse) {
- for(int j = 0; j != n; j++) {
- int i = sel[j];
- outputColVector.<OperatorMethod>(
- value,
- inputColVector2.asScratchPisaTimestamp(i),
- i);
- outputIsNull[i] = inputIsNull[i];
- }
- } else {
- for(int i = 0; i != n; i++) {
- outputColVector.<OperatorMethod>(
- value,
- inputColVector2.asScratchPisaTimestamp(i),
- i);
- }
- System.arraycopy(inputIsNull, 0, outputIsNull, 0, n);
- }
- }
-
- NullUtil.setNullOutputEntriesColScalar(outputColVector, batch.selectedInUse, sel, n);
- }
-
- @Override
- public int getOutputColumn() {
- return outputColumn;
- }
-
- @Override
- public String getOutputType() {
- return "timestamp";
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/FilterIntervalDayTimeColumnCompareIntervalDayTimeColumn.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/FilterIntervalDayTimeColumnCompareIntervalDayTimeColumn.txt b/ql/src/gen/vectorization/ExpressionTemplates/FilterIntervalDayTimeColumnCompareIntervalDayTimeColumn.txt
deleted file mode 100644
index 8d9bdf1..0000000
--- a/ql/src/gen/vectorization/ExpressionTemplates/FilterIntervalDayTimeColumnCompareIntervalDayTimeColumn.txt
+++ /dev/null
@@ -1,52 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
-
-import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
-
-/**
- * Generated from template FilterIntervalDayTimeColumnCompareColumn.txt, which covers comparison
- * expressions between a datetime/interval column and a scalar of the same type, however output is not
- * produced in a separate column.
- * The selected vector of the input {@link VectorizedRowBatch} is updated for in-place filtering.
- */
-public class <ClassName> extends <BaseClassName> {
-
- public <ClassName>(int colNum1, int colNum2) {
- super(colNum1, colNum2);
- }
-
- public <ClassName>() {
- super();
- }
-
- @Override
- public VectorExpressionDescriptor.Descriptor getDescriptor() {
- return (new VectorExpressionDescriptor.Builder())
- .setMode(
- VectorExpressionDescriptor.Mode.FILTER)
- .setNumArguments(2)
- .setArgumentTypes(
- VectorExpressionDescriptor.ArgumentType.getType("interval_day_time"),
- VectorExpressionDescriptor.ArgumentType.getType("interval_day_time"))
- .setInputExpressionTypes(
- VectorExpressionDescriptor.InputExpressionType.COLUMN,
- VectorExpressionDescriptor.InputExpressionType.COLUMN).build();
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/FilterIntervalDayTimeColumnCompareIntervalDayTimeScalar.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/FilterIntervalDayTimeColumnCompareIntervalDayTimeScalar.txt b/ql/src/gen/vectorization/ExpressionTemplates/FilterIntervalDayTimeColumnCompareIntervalDayTimeScalar.txt
deleted file mode 100644
index 7022b4f..0000000
--- a/ql/src/gen/vectorization/ExpressionTemplates/FilterIntervalDayTimeColumnCompareIntervalDayTimeScalar.txt
+++ /dev/null
@@ -1,55 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
-
-import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
-
-import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
-
-/**
- * Generated from template FilterIntervalDayTimeColumnCompareScalar.txt, which covers comparison
- * expressions between a datetime/interval column and a scalar of the same type, however output is not
- * produced in a separate column.
- * The selected vector of the input {@link VectorizedRowBatch} is updated for in-place filtering.
- */
-public class <ClassName> extends <BaseClassName> {
-
- public <ClassName>(int colNum, HiveIntervalDayTime value) {
- super(colNum, value.pisaTimestampUpdate(new PisaTimestamp()));
- }
-
- public <ClassName>() {
- super();
- }
-
- @Override
- public VectorExpressionDescriptor.Descriptor getDescriptor() {
- return (new VectorExpressionDescriptor.Builder())
- .setMode(
- VectorExpressionDescriptor.Mode.FILTER)
- .setNumArguments(2)
- .setArgumentTypes(
- VectorExpressionDescriptor.ArgumentType.getType("interval_day_time"),
- VectorExpressionDescriptor.ArgumentType.getType("interval_day_time"))
- .setInputExpressionTypes(
- VectorExpressionDescriptor.InputExpressionType.COLUMN,
- VectorExpressionDescriptor.InputExpressionType.SCALAR).build();
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/FilterIntervalDayTimeScalarCompareIntervalDayTimeColumn.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/FilterIntervalDayTimeScalarCompareIntervalDayTimeColumn.txt b/ql/src/gen/vectorization/ExpressionTemplates/FilterIntervalDayTimeScalarCompareIntervalDayTimeColumn.txt
deleted file mode 100644
index d227bf0..0000000
--- a/ql/src/gen/vectorization/ExpressionTemplates/FilterIntervalDayTimeScalarCompareIntervalDayTimeColumn.txt
+++ /dev/null
@@ -1,55 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
-
-import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
-
-import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
-
-/**
- * Generated from template FilterIntervalDayTimeScalarCompareColumn.txt, which covers comparison
- * expressions between a datetime/interval column and a scalar of the same type, however output is not
- * produced in a separate column.
- * The selected vector of the input {@link VectorizedRowBatch} is updated for in-place filtering.
- */
-public class <ClassName> extends <BaseClassName> {
-
- public <ClassName>(HiveIntervalDayTime value, int colNum) {
- super(value.pisaTimestampUpdate(new PisaTimestamp()), colNum);
- }
-
- public <ClassName>() {
- super();
- }
-
- @Override
- public VectorExpressionDescriptor.Descriptor getDescriptor() {
- return (new VectorExpressionDescriptor.Builder())
- .setMode(
- VectorExpressionDescriptor.Mode.FILTER)
- .setNumArguments(2)
- .setArgumentTypes(
- VectorExpressionDescriptor.ArgumentType.getType("interval_day_time"),
- VectorExpressionDescriptor.ArgumentType.getType("interval_day_time"))
- .setInputExpressionTypes(
- VectorExpressionDescriptor.InputExpressionType.SCALAR,
- VectorExpressionDescriptor.InputExpressionType.COLUMN).build();
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/FilterLongDoubleColumnCompareTimestampColumn.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/FilterLongDoubleColumnCompareTimestampColumn.txt b/ql/src/gen/vectorization/ExpressionTemplates/FilterLongDoubleColumnCompareTimestampColumn.txt
index 0c8321f..57caf7e 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/FilterLongDoubleColumnCompareTimestampColumn.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/FilterLongDoubleColumnCompareTimestampColumn.txt
@@ -19,8 +19,8 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
import java.sql.Timestamp;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil;
import org.apache.hadoop.hive.ql.exec.vector.*;
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/FilterLongDoubleColumnCompareTimestampScalar.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/FilterLongDoubleColumnCompareTimestampScalar.txt b/ql/src/gen/vectorization/ExpressionTemplates/FilterLongDoubleColumnCompareTimestampScalar.txt
index 7e4d55e..1b86691 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/FilterLongDoubleColumnCompareTimestampScalar.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/FilterLongDoubleColumnCompareTimestampScalar.txt
@@ -19,8 +19,8 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
import java.sql.Timestamp;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.*;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
@@ -36,7 +36,7 @@ public class <ClassName> extends <BaseClassName> {
private static final long serialVersionUID = 1L;
public <ClassName>(int colNum, Timestamp value) {
- super(colNum, new PisaTimestamp(value).<GetTimestampLongDoubleMethod>());
+ super(colNum, TimestampColumnVector.<GetTimestampLongDoubleMethod>(value));
}
public <ClassName>() {
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/FilterLongDoubleScalarCompareTimestampColumn.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/FilterLongDoubleScalarCompareTimestampColumn.txt b/ql/src/gen/vectorization/ExpressionTemplates/FilterLongDoubleScalarCompareTimestampColumn.txt
index ba6ca66..f5f59c2 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/FilterLongDoubleScalarCompareTimestampColumn.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/FilterLongDoubleScalarCompareTimestampColumn.txt
@@ -18,6 +18,10 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
+import java.sql.Timestamp;
+
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
+
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampColumnBetween.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampColumnBetween.txt b/ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampColumnBetween.txt
index 12f73da..4298d79 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampColumnBetween.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampColumnBetween.txt
@@ -20,7 +20,6 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
import java.sql.Timestamp;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
@@ -39,14 +38,14 @@ public class <ClassName> extends VectorExpression {
private int colNum;
// The comparison is of the form "column BETWEEN leftValue AND rightValue"
- private PisaTimestamp leftValue;
- private PisaTimestamp rightValue;
- private PisaTimestamp scratchValue;
+ private Timestamp leftValue;
+ private Timestamp rightValue;
+ private Timestamp scratchValue;
public <ClassName>(int colNum, Timestamp leftValue, Timestamp rightValue) {
this.colNum = colNum;
- this.leftValue = new PisaTimestamp(leftValue);
- this.rightValue = new PisaTimestamp(rightValue);
+ this.leftValue = leftValue;
+ this.rightValue = rightValue;
}
public <ClassName>() {
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampColumnCompareTimestampColumn.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampColumnCompareTimestampColumn.txt b/ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampColumnCompareTimestampColumn.txt
index 746b297..31dce1c 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampColumnCompareTimestampColumn.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampColumnCompareTimestampColumn.txt
@@ -18,22 +18,421 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
+import java.sql.Timestamp;
+
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
+import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.*;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
+import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
/**
- * Generated from template FilterTimestampColumnCompareTimestampColumn.txt, which covers comparison
- * expressions between a datetime/interval column and a scalar of the same type, however output is not
- * produced in a separate column.
+ * Generated from template FilterTimestampColumnCompareColumn.txt, which covers binary comparison
+ * filter expressions between two columns. Output is not produced in a separate column.
* The selected vector of the input {@link VectorizedRowBatch} is updated for in-place filtering.
*/
-public class <ClassName> extends <BaseClassName> {
+public class <ClassName> extends VectorExpression {
+
+ private static final long serialVersionUID = 1L;
- public <ClassName>(int colNum1, int colNum2) {
- super(colNum1, colNum2);
+ private int colNum1;
+ private int colNum2;
+
+ public <ClassName>(int colNum1, int colNum2) {
+ this.colNum1 = colNum1;
+ this.colNum2 = colNum2;
}
public <ClassName>() {
- super();
+ }
+
+ @Override
+ public void evaluate(VectorizedRowBatch batch) {
+
+ if (childExpressions != null) {
+ super.evaluateChildren(batch);
+ }
+
+ // Input #1 is type <OperandType>.
+ <InputColumnVectorType> inputColVector1 = (<InputColumnVectorType>) batch.cols[colNum1];
+
+ // Input #2 is type <OperandType>.
+ <InputColumnVectorType> inputColVector2 = (<InputColumnVectorType>) batch.cols[colNum2];
+
+ int[] sel = batch.selected;
+ boolean[] nullPos1 = inputColVector1.isNull;
+ boolean[] nullPos2 = inputColVector2.isNull;
+ int n = batch.size;
+
+ // return immediately if batch is empty
+ if (n == 0) {
+ return;
+ }
+
+ // handle case where neither input has nulls
+ if (inputColVector1.noNulls && inputColVector2.noNulls) {
+ if (inputColVector1.isRepeating && inputColVector2.isRepeating) {
+
+ /* Either all must remain selected or all will be eliminated.
+ * Repeating property will not change.
+ */
+ if (!(inputColVector1.compareTo(0, inputColVector2, 0) <OperatorSymbol> 0)) {
+ batch.size = 0;
+ }
+ } else if (inputColVector1.isRepeating) {
+ if (batch.selectedInUse) {
+ int newSize = 0;
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (inputColVector1.compareTo(0, inputColVector2, i) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ batch.size = newSize;
+ } else {
+ int newSize = 0;
+ for(int i = 0; i != n; i++) {
+ if (inputColVector1.compareTo(0, inputColVector2, i) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ if (newSize < batch.size) {
+ batch.size = newSize;
+ batch.selectedInUse = true;
+ }
+ }
+ } else if (inputColVector2.isRepeating) {
+ if (batch.selectedInUse) {
+ int newSize = 0;
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (inputColVector1.compareTo(i, inputColVector2, 0) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ batch.size = newSize;
+ } else {
+ int newSize = 0;
+ for(int i = 0; i != n; i++) {
+ if (inputColVector1.compareTo(i, inputColVector2, 0) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ if (newSize < batch.size) {
+ batch.size = newSize;
+ batch.selectedInUse = true;
+ }
+ }
+ } else if (batch.selectedInUse) {
+ int newSize = 0;
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (inputColVector1.compareTo(i, inputColVector2, i) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ batch.size = newSize;
+ } else {
+ int newSize = 0;
+ for(int i = 0; i != n; i++) {
+ if (inputColVector1.compareTo(i, inputColVector2, i) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ if (newSize < batch.size) {
+ batch.size = newSize;
+ batch.selectedInUse = true;
+ }
+ }
+
+ // handle case where only input 2 has nulls
+ } else if (inputColVector1.noNulls) {
+ if (inputColVector1.isRepeating && inputColVector2.isRepeating) {
+ if (nullPos2[0] ||
+ !(inputColVector1.compareTo(0, inputColVector2, 0) <OperatorSymbol> 0)) {
+ batch.size = 0;
+ }
+ } else if (inputColVector1.isRepeating) {
+
+ // no need to check for nulls in input 1
+ if (batch.selectedInUse) {
+ int newSize = 0;
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (!nullPos2[i]) {
+ if (inputColVector1.compareTo(0, inputColVector2, i) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ }
+ batch.size = newSize;
+ } else {
+ int newSize = 0;
+ for(int i = 0; i != n; i++) {
+ if (!nullPos2[i]) {
+ if (inputColVector1.compareTo(0, inputColVector2, i) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ }
+ if (newSize < batch.size) {
+ batch.size = newSize;
+ batch.selectedInUse = true;
+ }
+ }
+ } else if (inputColVector2.isRepeating) {
+ if (nullPos2[0]) {
+
+ // no values will qualify because every comparison will be with NULL
+ batch.size = 0;
+ return;
+ }
+ if (batch.selectedInUse) {
+ int newSize = 0;
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (inputColVector1.compareTo(i, inputColVector2, 0) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ batch.size = newSize;
+ } else {
+ int newSize = 0;
+ for(int i = 0; i != n; i++) {
+ if (inputColVector1.compareTo(i, inputColVector2, 0) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ if (newSize < batch.size) {
+ batch.size = newSize;
+ batch.selectedInUse = true;
+ }
+ }
+ } else { // neither input is repeating
+ if (batch.selectedInUse) {
+ int newSize = 0;
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (!nullPos2[i]) {
+ if (inputColVector1.compareTo(i, inputColVector2, i) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ }
+ batch.size = newSize;
+ } else {
+ int newSize = 0;
+ for(int i = 0; i != n; i++) {
+ if (!nullPos2[i]) {
+ if (inputColVector1.compareTo(i, inputColVector2, i) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ }
+ if (newSize < batch.size) {
+ batch.size = newSize;
+ batch.selectedInUse = true;
+ }
+ }
+ }
+
+ // handle case where only input 1 has nulls
+ } else if (inputColVector2.noNulls) {
+ if (inputColVector1.isRepeating && inputColVector2.isRepeating) {
+ if (nullPos1[0] ||
+ !(inputColVector1.compareTo(0, inputColVector2, 0) <OperatorSymbol> 0)) {
+ batch.size = 0;
+ return;
+ }
+ } else if (inputColVector1.isRepeating) {
+ if (nullPos1[0]) {
+
+ // if repeating value is null then every comparison will fail so nothing qualifies
+ batch.size = 0;
+ return;
+ }
+ if (batch.selectedInUse) {
+ int newSize = 0;
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (inputColVector1.compareTo(0, inputColVector2, i) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ batch.size = newSize;
+ } else {
+ int newSize = 0;
+ for(int i = 0; i != n; i++) {
+ if (inputColVector1.compareTo(0, inputColVector2, i) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ if (newSize < batch.size) {
+ batch.size = newSize;
+ batch.selectedInUse = true;
+ }
+ }
+ } else if (inputColVector2.isRepeating) {
+ if (batch.selectedInUse) {
+ int newSize = 0;
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (!nullPos1[i]) {
+ if (inputColVector1.compareTo(i, inputColVector2, 0) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ }
+ batch.size = newSize;
+ } else {
+ int newSize = 0;
+ for(int i = 0; i != n; i++) {
+ if (!nullPos1[i]) {
+ if (inputColVector1.compareTo(i, inputColVector2, 0) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ }
+ if (newSize < batch.size) {
+ batch.size = newSize;
+ batch.selectedInUse = true;
+ }
+ }
+ } else { // neither input is repeating
+ if (batch.selectedInUse) {
+ int newSize = 0;
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (!nullPos1[i]) {
+ if (inputColVector1.compareTo(i, inputColVector2, i) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ }
+ batch.size = newSize;
+ } else {
+ int newSize = 0;
+ for(int i = 0; i != n; i++) {
+ if (!nullPos1[i]) {
+ if (inputColVector1.compareTo(i, inputColVector2, i) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ }
+ if (newSize < batch.size) {
+ batch.size = newSize;
+ batch.selectedInUse = true;
+ }
+ }
+ }
+
+ // handle case where both inputs have nulls
+ } else {
+ if (inputColVector1.isRepeating && inputColVector2.isRepeating) {
+ if (nullPos1[0] || nullPos2[0] ||
+ !(inputColVector1.compareTo(0, inputColVector2, 0) <OperatorSymbol> 0)) {
+ batch.size = 0;
+ }
+ } else if (inputColVector1.isRepeating) {
+ if (nullPos1[0]) {
+ batch.size = 0;
+ return;
+ }
+ if (batch.selectedInUse) {
+ int newSize = 0;
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (!nullPos2[i]) {
+ if (inputColVector1.compareTo(0, inputColVector2, i) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ }
+ batch.size = newSize;
+ } else {
+ int newSize = 0;
+ for(int i = 0; i != n; i++) {
+ if (!nullPos2[i]) {
+ if (inputColVector1.compareTo(0, inputColVector2, i) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ }
+ if (newSize < batch.size) {
+ batch.size = newSize;
+ batch.selectedInUse = true;
+ }
+ }
+ } else if (inputColVector2.isRepeating) {
+ if (nullPos2[0]) {
+ batch.size = 0;
+ return;
+ }
+ if (batch.selectedInUse) {
+ int newSize = 0;
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (!nullPos1[i]) {
+ if (inputColVector1.compareTo(i, inputColVector2, 0) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ }
+ batch.size = newSize;
+ } else {
+ int newSize = 0;
+ for(int i = 0; i != n; i++) {
+ if (!nullPos1[i]) {
+ if (inputColVector1.compareTo(i, inputColVector2, 0) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ }
+ if (newSize < batch.size) {
+ batch.size = newSize;
+ batch.selectedInUse = true;
+ }
+ }
+ } else { // neither input is repeating
+ if (batch.selectedInUse) {
+ int newSize = 0;
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (!nullPos1[i] && !nullPos2[i]) {
+ if (inputColVector1.compareTo(i, inputColVector2, i) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ }
+ batch.size = newSize;
+ } else {
+ int newSize = 0;
+ for(int i = 0; i != n; i++) {
+ if (!nullPos1[i] && !nullPos2[i]) {
+ if (inputColVector1.compareTo(i, inputColVector2, i) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ }
+ if (newSize < batch.size) {
+ batch.size = newSize;
+ batch.selectedInUse = true;
+ }
+ }
+ }
+ }
+ }
+
+ @Override
+ public String getOutputType() {
+ return "boolean";
+ }
+
+ @Override
+ public int getOutputColumn() {
+ return -1;
}
@Override
@@ -43,8 +442,8 @@ public class <ClassName> extends <BaseClassName> {
VectorExpressionDescriptor.Mode.FILTER)
.setNumArguments(2)
.setArgumentTypes(
- VectorExpressionDescriptor.ArgumentType.getType("timestamp"),
- VectorExpressionDescriptor.ArgumentType.getType("timestamp"))
+ VectorExpressionDescriptor.ArgumentType.getType("<OperandType>"),
+ VectorExpressionDescriptor.ArgumentType.getType("<OperandType>"))
.setInputExpressionTypes(
VectorExpressionDescriptor.InputExpressionType.COLUMN,
VectorExpressionDescriptor.InputExpressionType.COLUMN).build();
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampColumnCompareTimestampColumnBase.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampColumnCompareTimestampColumnBase.txt b/ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampColumnCompareTimestampColumnBase.txt
deleted file mode 100644
index b5a7a7a..0000000
--- a/ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampColumnCompareTimestampColumnBase.txt
+++ /dev/null
@@ -1,429 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
-
-import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
-import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
-import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
-import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
-
-/**
- * Generated from template FilterTimestampColumnCompareColumn.txt, which covers binary comparison
- * filter expressions between two columns. Output is not produced in a separate column.
- * The selected vector of the input {@link VectorizedRowBatch} is updated for in-place filtering.
- */
-public abstract class <ClassName> extends VectorExpression {
-
- private static final long serialVersionUID = 1L;
-
- private int colNum1;
- private int colNum2;
-
- public <ClassName>(int colNum1, int colNum2) {
- this.colNum1 = colNum1;
- this.colNum2 = colNum2;
- }
-
- public <ClassName>() {
- }
-
- @Override
- public void evaluate(VectorizedRowBatch batch) {
-
- if (childExpressions != null) {
- super.evaluateChildren(batch);
- }
-
- TimestampColumnVector inputColVector1 = (TimestampColumnVector) batch.cols[colNum1];
- TimestampColumnVector inputColVector2 = (TimestampColumnVector) batch.cols[colNum2];
- int[] sel = batch.selected;
- boolean[] nullPos1 = inputColVector1.isNull;
- boolean[] nullPos2 = inputColVector2.isNull;
- int n = batch.size;
-
- // return immediately if batch is empty
- if (n == 0) {
- return;
- }
-
- // handle case where neither input has nulls
- if (inputColVector1.noNulls && inputColVector2.noNulls) {
- if (inputColVector1.isRepeating && inputColVector2.isRepeating) {
-
- /* Either all must remain selected or all will be eliminated.
- * Repeating property will not change.
- */
- if (!(inputColVector1.compareTo(0, inputColVector2, 0) <OperatorSymbol> 0)) {
- batch.size = 0;
- }
- } else if (inputColVector1.isRepeating) {
- if (batch.selectedInUse) {
- int newSize = 0;
- for(int j = 0; j != n; j++) {
- int i = sel[j];
- if (inputColVector1.compareTo(0, inputColVector2, i) <OperatorSymbol> 0) {
- sel[newSize++] = i;
- }
- }
- batch.size = newSize;
- } else {
- int newSize = 0;
- for(int i = 0; i != n; i++) {
- if (inputColVector1.compareTo(0, inputColVector2, i) <OperatorSymbol> 0) {
- sel[newSize++] = i;
- }
- }
- if (newSize < batch.size) {
- batch.size = newSize;
- batch.selectedInUse = true;
- }
- }
- } else if (inputColVector2.isRepeating) {
- if (batch.selectedInUse) {
- int newSize = 0;
- for(int j = 0; j != n; j++) {
- int i = sel[j];
- if (inputColVector1.compareTo(i, inputColVector2, 0) <OperatorSymbol> 0) {
- sel[newSize++] = i;
- }
- }
- batch.size = newSize;
- } else {
- int newSize = 0;
- for(int i = 0; i != n; i++) {
- if (inputColVector1.compareTo(i, inputColVector2, 0) <OperatorSymbol> 0) {
- sel[newSize++] = i;
- }
- }
- if (newSize < batch.size) {
- batch.size = newSize;
- batch.selectedInUse = true;
- }
- }
- } else if (batch.selectedInUse) {
- int newSize = 0;
- for(int j = 0; j != n; j++) {
- int i = sel[j];
- if (inputColVector1.compareTo(i, inputColVector2, i) <OperatorSymbol> 0) {
- sel[newSize++] = i;
- }
- }
- batch.size = newSize;
- } else {
- int newSize = 0;
- for(int i = 0; i != n; i++) {
- if (inputColVector1.compareTo(i, inputColVector2, i) <OperatorSymbol> 0) {
- sel[newSize++] = i;
- }
- }
- if (newSize < batch.size) {
- batch.size = newSize;
- batch.selectedInUse = true;
- }
- }
-
- // handle case where only input 2 has nulls
- } else if (inputColVector1.noNulls) {
- if (inputColVector1.isRepeating && inputColVector2.isRepeating) {
- if (nullPos2[0] ||
- !(inputColVector1.compareTo(0, inputColVector2, 0) <OperatorSymbol> 0)) {
- batch.size = 0;
- }
- } else if (inputColVector1.isRepeating) {
-
- // no need to check for nulls in input 1
- if (batch.selectedInUse) {
- int newSize = 0;
- for(int j = 0; j != n; j++) {
- int i = sel[j];
- if (!nullPos2[i]) {
- if (inputColVector1.compareTo(0, inputColVector2, i) <OperatorSymbol> 0) {
- sel[newSize++] = i;
- }
- }
- }
- batch.size = newSize;
- } else {
- int newSize = 0;
- for(int i = 0; i != n; i++) {
- if (!nullPos2[i]) {
- if (inputColVector1.compareTo(0, inputColVector2, i) <OperatorSymbol> 0) {
- sel[newSize++] = i;
- }
- }
- }
- if (newSize < batch.size) {
- batch.size = newSize;
- batch.selectedInUse = true;
- }
- }
- } else if (inputColVector2.isRepeating) {
- if (nullPos2[0]) {
-
- // no values will qualify because every comparison will be with NULL
- batch.size = 0;
- return;
- }
- if (batch.selectedInUse) {
- int newSize = 0;
- for(int j = 0; j != n; j++) {
- int i = sel[j];
- if (inputColVector1.compareTo(i, inputColVector2, 0) <OperatorSymbol> 0) {
- sel[newSize++] = i;
- }
- }
- batch.size = newSize;
- } else {
- int newSize = 0;
- for(int i = 0; i != n; i++) {
- if (inputColVector1.compareTo(i, inputColVector2, 0) <OperatorSymbol> 0) {
- sel[newSize++] = i;
- }
- }
- if (newSize < batch.size) {
- batch.size = newSize;
- batch.selectedInUse = true;
- }
- }
- } else { // neither input is repeating
- if (batch.selectedInUse) {
- int newSize = 0;
- for(int j = 0; j != n; j++) {
- int i = sel[j];
- if (!nullPos2[i]) {
- if (inputColVector1.compareTo(i, inputColVector2, i) <OperatorSymbol> 0) {
- sel[newSize++] = i;
- }
- }
- }
- batch.size = newSize;
- } else {
- int newSize = 0;
- for(int i = 0; i != n; i++) {
- if (!nullPos2[i]) {
- if (inputColVector1.compareTo(i, inputColVector2, i) <OperatorSymbol> 0) {
- sel[newSize++] = i;
- }
- }
- }
- if (newSize < batch.size) {
- batch.size = newSize;
- batch.selectedInUse = true;
- }
- }
- }
-
- // handle case where only input 1 has nulls
- } else if (inputColVector2.noNulls) {
- if (inputColVector1.isRepeating && inputColVector2.isRepeating) {
- if (nullPos1[0] ||
- !(inputColVector1.compareTo(0, inputColVector2, 0) <OperatorSymbol> 0)) {
- batch.size = 0;
- return;
- }
- } else if (inputColVector1.isRepeating) {
- if (nullPos1[0]) {
-
- // if repeating value is null then every comparison will fail so nothing qualifies
- batch.size = 0;
- return;
- }
- if (batch.selectedInUse) {
- int newSize = 0;
- for(int j = 0; j != n; j++) {
- int i = sel[j];
- if (inputColVector1.compareTo(0, inputColVector2, i) <OperatorSymbol> 0) {
- sel[newSize++] = i;
- }
- }
- batch.size = newSize;
- } else {
- int newSize = 0;
- for(int i = 0; i != n; i++) {
- if (inputColVector1.compareTo(0, inputColVector2, i) <OperatorSymbol> 0) {
- sel[newSize++] = i;
- }
- }
- if (newSize < batch.size) {
- batch.size = newSize;
- batch.selectedInUse = true;
- }
- }
- } else if (inputColVector2.isRepeating) {
- if (batch.selectedInUse) {
- int newSize = 0;
- for(int j = 0; j != n; j++) {
- int i = sel[j];
- if (!nullPos1[i]) {
- if (inputColVector1.compareTo(i, inputColVector2, 0) <OperatorSymbol> 0) {
- sel[newSize++] = i;
- }
- }
- }
- batch.size = newSize;
- } else {
- int newSize = 0;
- for(int i = 0; i != n; i++) {
- if (!nullPos1[i]) {
- if (inputColVector1.compareTo(i, inputColVector2, 0) <OperatorSymbol> 0) {
- sel[newSize++] = i;
- }
- }
- }
- if (newSize < batch.size) {
- batch.size = newSize;
- batch.selectedInUse = true;
- }
- }
- } else { // neither input is repeating
- if (batch.selectedInUse) {
- int newSize = 0;
- for(int j = 0; j != n; j++) {
- int i = sel[j];
- if (!nullPos1[i]) {
- if (inputColVector1.compareTo(i, inputColVector2, i) <OperatorSymbol> 0) {
- sel[newSize++] = i;
- }
- }
- }
- batch.size = newSize;
- } else {
- int newSize = 0;
- for(int i = 0; i != n; i++) {
- if (!nullPos1[i]) {
- if (inputColVector1.compareTo(i, inputColVector2, i) <OperatorSymbol> 0) {
- sel[newSize++] = i;
- }
- }
- }
- if (newSize < batch.size) {
- batch.size = newSize;
- batch.selectedInUse = true;
- }
- }
- }
-
- // handle case where both inputs have nulls
- } else {
- if (inputColVector1.isRepeating && inputColVector2.isRepeating) {
- if (nullPos1[0] || nullPos2[0] ||
- !(inputColVector1.compareTo(0, inputColVector2, 0) <OperatorSymbol> 0)) {
- batch.size = 0;
- }
- } else if (inputColVector1.isRepeating) {
- if (nullPos1[0]) {
- batch.size = 0;
- return;
- }
- if (batch.selectedInUse) {
- int newSize = 0;
- for(int j = 0; j != n; j++) {
- int i = sel[j];
- if (!nullPos2[i]) {
- if (inputColVector1.compareTo(0, inputColVector2, i) <OperatorSymbol> 0) {
- sel[newSize++] = i;
- }
- }
- }
- batch.size = newSize;
- } else {
- int newSize = 0;
- for(int i = 0; i != n; i++) {
- if (!nullPos2[i]) {
- if (inputColVector1.compareTo(0, inputColVector2, i) <OperatorSymbol> 0) {
- sel[newSize++] = i;
- }
- }
- }
- if (newSize < batch.size) {
- batch.size = newSize;
- batch.selectedInUse = true;
- }
- }
- } else if (inputColVector2.isRepeating) {
- if (nullPos2[0]) {
- batch.size = 0;
- return;
- }
- if (batch.selectedInUse) {
- int newSize = 0;
- for(int j = 0; j != n; j++) {
- int i = sel[j];
- if (!nullPos1[i]) {
- if (inputColVector1.compareTo(i, inputColVector2, 0) <OperatorSymbol> 0) {
- sel[newSize++] = i;
- }
- }
- }
- batch.size = newSize;
- } else {
- int newSize = 0;
- for(int i = 0; i != n; i++) {
- if (!nullPos1[i]) {
- if (inputColVector1.compareTo(i, inputColVector2, 0) <OperatorSymbol> 0) {
- sel[newSize++] = i;
- }
- }
- }
- if (newSize < batch.size) {
- batch.size = newSize;
- batch.selectedInUse = true;
- }
- }
- } else { // neither input is repeating
- if (batch.selectedInUse) {
- int newSize = 0;
- for(int j = 0; j != n; j++) {
- int i = sel[j];
- if (!nullPos1[i] && !nullPos2[i]) {
- if (inputColVector1.compareTo(i, inputColVector2, i) <OperatorSymbol> 0) {
- sel[newSize++] = i;
- }
- }
- }
- batch.size = newSize;
- } else {
- int newSize = 0;
- for(int i = 0; i != n; i++) {
- if (!nullPos1[i] && !nullPos2[i]) {
- if (inputColVector1.compareTo(i, inputColVector2, i) <OperatorSymbol> 0) {
- sel[newSize++] = i;
- }
- }
- }
- if (newSize < batch.size) {
- batch.size = newSize;
- batch.selectedInUse = true;
- }
- }
- }
- }
- }
-
- @Override
- public String getOutputType() {
- return "boolean";
- }
-
- @Override
- public int getOutputColumn() {
- return -1;
- }
-}
[36/50] [abbrv] hive git commit: HIVE-13371: Fix test failure of
testHasNull in TestColumnStatistics running on Windows (PPengcheng Xiong,
reviewed by Ashutosh Chauhan)
Posted by jd...@apache.org.
HIVE-13371: Fix test failure of testHasNull in TestColumnStatistics running on Windows (PPengcheng Xiong, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/44ab4553
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/44ab4553
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/44ab4553
Branch: refs/heads/llap
Commit: 44ab45534277920bcf64dbd43409ab730fdb8d61
Parents: 5201629
Author: Pengcheng Xiong <px...@apache.org>
Authored: Mon Mar 28 16:04:45 2016 -0700
Committer: Pengcheng Xiong <px...@apache.org>
Committed: Mon Mar 28 16:04:45 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/ql/io/orc/TestColumnStatistics.java | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/44ab4553/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestColumnStatistics.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestColumnStatistics.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestColumnStatistics.java
index 9433283..5f0146f 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestColumnStatistics.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestColumnStatistics.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hive.ql.io.orc;
import static junit.framework.Assert.assertEquals;
+import static org.junit.Assume.assumeTrue;
import java.io.File;
import java.io.FileOutputStream;
@@ -343,7 +344,9 @@ public class TestColumnStatistics {
FileDump.main(new String[]{testFilePath.toString(), "--rowindex=2"});
System.out.flush();
System.setOut(origOut);
-
+ // If called with an expression evaluating to false, the test will halt
+ // and be ignored.
+ assumeTrue(!System.getProperty("os.name").startsWith("Windows"));
TestFileDump.checkOutput(outputFilename, workDir + File.separator + outputFilename);
}
}
[20/50] [abbrv] hive git commit: HIVE-13115: MetaStore Direct SQL
getPartitions call fail when the columns schemas for a partition are null
(Ratandeep Ratti reviewed by Carl Steinbach)
Posted by jd...@apache.org.
HIVE-13115: MetaStore Direct SQL getPartitions call fail when the columns schemas for a partition are null (Ratandeep Ratti reviewed by Carl Steinbach)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/69cfd357
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/69cfd357
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/69cfd357
Branch: refs/heads/llap
Commit: 69cfd357eb482c426161aa3c4a00d574ee29416e
Parents: 9686209
Author: Carl Steinbach <cw...@apache.org>
Authored: Sun Mar 27 15:41:38 2016 -0700
Committer: Carl Steinbach <cw...@apache.org>
Committed: Sun Mar 27 15:41:38 2016 -0700
----------------------------------------------------------------------
.../hive/metastore/TestHiveMetaStore.java | 20 +++++++++++-
.../hive/metastore/MetaStoreDirectSql.java | 32 +++++++++++---------
2 files changed, 37 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/69cfd357/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
index 5da4165..83fb15c 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
@@ -280,6 +280,24 @@ public abstract class TestHiveMetaStore extends TestCase {
}
assertTrue("Partitions are not same", part.equals(part_get));
+ // check null cols schemas for a partition
+ List<String> vals6 = makeVals("2016-02-22 00:00:00", "16");
+ Partition part6 = makePartitionObject(dbName, tblName, vals6, tbl, "/part5");
+ part6.getSd().setCols(null);
+ LOG.info("Creating partition will null field schema");
+ client.add_partition(part6);
+ LOG.info("Listing all partitions for table " + dbName + "." + tblName);
+ final List<Partition> partitions = client.listPartitions(dbName, tblName, (short) -1);
+ boolean foundPart = false;
+ for (Partition p : partitions) {
+ if (p.getValues().equals(vals6)) {
+ assertNull(p.getSd().getCols());
+ LOG.info("Found partition " + p + " having null field schema");
+ foundPart = true;
+ }
+ }
+ assertTrue(foundPart);
+
String partName = "ds=" + FileUtils.escapePathName("2008-07-01 14:13:12") + "/hr=14";
String part2Name = "ds=" + FileUtils.escapePathName("2008-07-01 14:13:12") + "/hr=15";
String part3Name = "ds=" + FileUtils.escapePathName("2008-07-02 14:13:12") + "/hr=15";
@@ -313,7 +331,7 @@ public abstract class TestHiveMetaStore extends TestCase {
partialVals.clear();
partialVals.add("");
partialNames = client.listPartitionNames(dbName, tblName, partialVals, (short) -1);
- assertTrue("Should have returned 4 partition names", partialNames.size() == 4);
+ assertTrue("Should have returned 5 partition names", partialNames.size() == 5);
assertTrue("Not all part names returned", partialNames.containsAll(partNames));
// Test partition listing with a partial spec - hr is specified but ds is not
http://git-wip-us.apache.org/repos/asf/hive/blob/69cfd357/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
index d51f58d..06e9f78 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
@@ -557,14 +557,14 @@ class MetaStoreDirectSql {
Long sdId = extractSqlLong(fields[1]);
Long colId = extractSqlLong(fields[2]);
Long serdeId = extractSqlLong(fields[3]);
- // A partition must have either everything set, or nothing set if it's a view.
- if (sdId == null || colId == null || serdeId == null) {
+ // A partition must have at least sdId and serdeId set, or nothing set if it's a view.
+ if (sdId == null || serdeId == null) {
if (isView == null) {
isView = isViewTable(dbName, tblName);
}
if ((sdId != null || colId != null || serdeId != null) || !isView) {
- throw new MetaException("Unexpected null for one of the IDs, SD " + sdId + ", column "
- + colId + ", serde " + serdeId + " for a " + (isView ? "" : "non-") + " view");
+ throw new MetaException("Unexpected null for one of the IDs, SD " + sdId +
+ ", serde " + serdeId + " for a " + (isView ? "" : "non-") + " view");
}
}
@@ -580,7 +580,7 @@ class MetaStoreDirectSql {
partitions.put(partitionId, part);
if (sdId == null) continue; // Probably a view.
- assert colId != null && serdeId != null;
+ assert serdeId != null;
// We assume each partition has an unique SD.
StorageDescriptor sd = new StorageDescriptor();
@@ -605,14 +605,16 @@ class MetaStoreDirectSql {
sdSb.append(sdId).append(",");
part.setSd(sd);
- List<FieldSchema> cols = colss.get(colId);
- // We expect that colId will be the same for all (or many) SDs.
- if (cols == null) {
- cols = new ArrayList<FieldSchema>();
- colss.put(colId, cols);
- colsSb.append(colId).append(",");
+ if (colId != null) {
+ List<FieldSchema> cols = colss.get(colId);
+ // We expect that colId will be the same for all (or many) SDs.
+ if (cols == null) {
+ cols = new ArrayList<FieldSchema>();
+ colss.put(colId, cols);
+ colsSb.append(colId).append(",");
+ }
+ sd.setCols(cols);
}
- sd.setCols(cols);
// We assume each SD has an unique serde.
SerDeInfo serde = new SerDeInfo();
@@ -658,8 +660,10 @@ class MetaStoreDirectSql {
assert serdeSb.length() == 0 && colsSb.length() == 0;
return orderedResult; // No SDs, probably a view.
}
- String sdIds = trimCommaList(sdSb), serdeIds = trimCommaList(serdeSb),
- colIds = trimCommaList(colsSb);
+
+ String sdIds = trimCommaList(sdSb);
+ String serdeIds = trimCommaList(serdeSb);
+ String colIds = trimCommaList(colsSb);
// Get all the stuff for SD. Don't do empty-list check - we expect partitions do have SDs.
queryText = "select \"SD_ID\", \"PARAM_KEY\", \"PARAM_VALUE\" from \"SD_PARAMS\""
[16/50] [abbrv] hive git commit: HIVE-12653 : The property
"serialization.encoding" in the class
"org.apache.hadoop.hive.contrib.serde2.MultiDelimitSerDe" does not work
(yangfeng via Ashutosh Chauhan)
Posted by jd...@apache.org.
HIVE-12653 : The property "serialization.encoding" in the class "org.apache.hadoop.hive.contrib.serde2.MultiDelimitSerDe" does not work (yangfeng via Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2449d1df
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2449d1df
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2449d1df
Branch: refs/heads/llap
Commit: 2449d1dfe9429363a9458d2004ec2405f5aa9035
Parents: e384b2b
Author: yangfang <ya...@zte.com.cn>
Authored: Mon Dec 14 03:13:00 2015 -0800
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Fri Mar 25 07:43:05 2016 -0700
----------------------------------------------------------------------
.../hive/contrib/serde2/MultiDelimitSerDe.java | 23 +++++++++++++++-----
1 file changed, 18 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/2449d1df/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/MultiDelimitSerDe.java
----------------------------------------------------------------------
diff --git a/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/MultiDelimitSerDe.java b/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/MultiDelimitSerDe.java
index 9a162d5..296c449 100644
--- a/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/MultiDelimitSerDe.java
+++ b/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/MultiDelimitSerDe.java
@@ -63,7 +63,7 @@ import org.apache.hadoop.io.Writable;
serdeConstants.SERIALIZATION_ENCODING,
LazySerDeParameters.SERIALIZATION_EXTEND_NESTING_LEVELS,
LazySerDeParameters.SERIALIZATION_EXTEND_ADDITIONAL_NESTING_LEVELS})
-public class MultiDelimitSerDe extends AbstractSerDe {
+public class MultiDelimitSerDe extends AbstractEncodingAwareSerDe {
private static final byte[] DEFAULT_SEPARATORS = {(byte) 1, (byte) 2, (byte) 3};
// Due to HIVE-6404, define our own constant
@@ -94,6 +94,7 @@ public class MultiDelimitSerDe extends AbstractSerDe {
@Override
public void initialize(Configuration conf, Properties tbl) throws SerDeException {
// get the SerDe parameters
+ super.initialize(conf, tbl);
serdeParams = new LazySerDeParameters(conf, tbl, getClass().getName());
fieldDelimited = tbl.getProperty(serdeConstants.FIELD_DELIM);
@@ -134,8 +135,9 @@ public class MultiDelimitSerDe extends AbstractSerDe {
return Text.class;
}
- @Override
- public Object deserialize(Writable blob) throws SerDeException {
+
+ @Override
+ public Object doDeserialize(Writable blob) throws SerDeException {
if (byteArrayRef == null) {
byteArrayRef = new ByteArrayRef();
}
@@ -159,8 +161,9 @@ public class MultiDelimitSerDe extends AbstractSerDe {
return cachedLazyStruct;
}
- @Override
- public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException {
+ @Override
+ public Writable doSerialize(Object obj, ObjectInspector objInspector)
+ throws SerDeException {
StructObjectInspector soi = (StructObjectInspector) objInspector;
List<? extends StructField> fields = soi.getAllStructFieldRefs();
List<Object> list = soi.getStructFieldsDataAsList(obj);
@@ -286,6 +289,16 @@ public class MultiDelimitSerDe extends AbstractSerDe {
throw new RuntimeException("Unknown category type: "+ objInspector.getCategory());
}
+ protected Text transformFromUTF8(Writable blob) {
+ Text text = (Text)blob;
+ return SerDeUtils.transformTextFromUTF8(text, this.charset);
+ }
+
+ protected Text transformToUTF8(Writable blob) {
+ Text text = (Text) blob;
+ return SerDeUtils.transformTextToUTF8(text, this.charset);
+ }
+
@Override
public SerDeStats getSerDeStats() {
// no support for statistics
[29/50] [abbrv] hive git commit: HIVE-13111: Fix timestamp /
interval_day_time wrong results with HIVE-9862 (Matt McCline,
reviewed by Jason Dere)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/TimestampColumnInList.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/TimestampColumnInList.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/TimestampColumnInList.java
index 2d7d0c2..bc09a3a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/TimestampColumnInList.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/TimestampColumnInList.java
@@ -21,7 +21,6 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions;
import java.sql.Timestamp;
import java.util.HashSet;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor.Descriptor;
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
@@ -36,11 +35,8 @@ public class TimestampColumnInList extends VectorExpression implements ITimestam
private Timestamp[] inListValues;
private int outputColumn;
- private transient PisaTimestamp scratchTimestamp;
-
-
// The set object containing the IN list.
- private transient HashSet<PisaTimestamp> inSet;
+ private transient HashSet<Timestamp> inSet;
public TimestampColumnInList() {
super();
@@ -64,11 +60,10 @@ public class TimestampColumnInList extends VectorExpression implements ITimestam
}
if (inSet == null) {
- inSet = new HashSet<PisaTimestamp>(inListValues.length);
+ inSet = new HashSet<Timestamp>(inListValues.length);
for (Timestamp val : inListValues) {
- inSet.add(new PisaTimestamp(val));
+ inSet.add(val);
}
- scratchTimestamp = new PisaTimestamp();
}
TimestampColumnVector inputColVector = (TimestampColumnVector) batch.cols[inputCol];
@@ -91,19 +86,16 @@ public class TimestampColumnInList extends VectorExpression implements ITimestam
// All must be selected otherwise size would be zero
// Repeating property will not change.
- inputColVector.pisaTimestampUpdate(scratchTimestamp, 0);
- outputVector[0] = inSet.contains(scratchTimestamp) ? 1 : 0;
+ outputVector[0] = inSet.contains(inputColVector.asScratchTimestamp(0)) ? 1 : 0;
outputColVector.isRepeating = true;
} else if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- inputColVector.pisaTimestampUpdate(scratchTimestamp, i);
- outputVector[i] = inSet.contains(scratchTimestamp) ? 1 : 0;
+ outputVector[i] = inSet.contains(inputColVector.asScratchTimestamp(i)) ? 1 : 0;
}
} else {
for(int i = 0; i != n; i++) {
- inputColVector.pisaTimestampUpdate(scratchTimestamp, i);
- outputVector[i] = inSet.contains(scratchTimestamp) ? 1 : 0;
+ outputVector[i] = inSet.contains(inputColVector.asScratchTimestamp(i)) ? 1 : 0;
}
}
} else {
@@ -112,8 +104,7 @@ public class TimestampColumnInList extends VectorExpression implements ITimestam
//All must be selected otherwise size would be zero
//Repeating property will not change.
if (!nullPos[0]) {
- inputColVector.pisaTimestampUpdate(scratchTimestamp, 0);
- outputVector[0] = inSet.contains(scratchTimestamp) ? 1 : 0;
+ outputVector[0] = inSet.contains(inputColVector.asScratchTimestamp(0)) ? 1 : 0;
outNulls[0] = false;
} else {
outNulls[0] = true;
@@ -124,16 +115,14 @@ public class TimestampColumnInList extends VectorExpression implements ITimestam
int i = sel[j];
outNulls[i] = nullPos[i];
if (!nullPos[i]) {
- inputColVector.pisaTimestampUpdate(scratchTimestamp, i);
- outputVector[i] = inSet.contains(scratchTimestamp) ? 1 : 0;
+ outputVector[i] = inSet.contains(inputColVector.asScratchTimestamp(i)) ? 1 : 0;
}
}
} else {
System.arraycopy(nullPos, 0, outNulls, 0, n);
for(int i = 0; i != n; i++) {
if (!nullPos[i]) {
- inputColVector.pisaTimestampUpdate(scratchTimestamp, i);
- outputVector[i] = inSet.contains(scratchTimestamp) ? 1 : 0;
+ outputVector[i] = inSet.contains(inputColVector.asScratchTimestamp(i)) ? 1 : 0;
}
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriter.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriter.java
index 326bfb9..85dacd7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriter.java
@@ -21,10 +21,11 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions;
import java.sql.Timestamp;
import org.apache.hadoop.hive.common.type.HiveDecimal;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
+import org.apache.hadoop.hive.serde2.io.HiveIntervalDayTimeWritable;
import org.apache.hadoop.hive.serde2.io.TimestampWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -42,7 +43,8 @@ public interface VectorExpressionWriter {
Object writeValue(HiveDecimal value) throws HiveException;
Object writeValue(TimestampWritable value) throws HiveException;
Object writeValue(Timestamp value) throws HiveException;
- Object writeValue(PisaTimestamp value) throws HiveException;
+ Object writeValue(HiveIntervalDayTimeWritable value) throws HiveException;
+ Object writeValue(HiveIntervalDayTime value) throws HiveException;
Object setValue(Object row, ColumnVector column, int columnRow) throws HiveException;
Object initValue(Object ost) throws HiveException;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java
index 9a1d7f3..c20bc68 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java
@@ -31,7 +31,6 @@ import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
import org.apache.hadoop.hive.common.type.HiveVarchar;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
import org.apache.hadoop.hive.ql.exec.vector.*;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
@@ -188,17 +187,39 @@ public final class VectorExpressionWriterFactory {
}
/**
- * The base implementation must be overridden by the PisaTimestamp specialization
+ * The base implementation must be overridden by the Timestamp specialization
+ */
+ public Object setValue(Object field, Timestamp value) throws HiveException {
+ throw new HiveException("Internal error: should not reach here");
+ }
+
+ /**
+ * The base implementation must be overridden by the HiveIntervalDayTime specialization
*/
@Override
- public Object writeValue(PisaTimestamp value) throws HiveException {
+ public Object writeValue(HiveIntervalDayTimeWritable value) throws HiveException {
throw new HiveException("Internal error: should not reach here");
}
/**
- * The base implementation must be overridden by the Timestamp specialization
+ * The base implementation must be overridden by the HiveIntervalDayTime specialization
*/
- public Object setValue(Object field, Timestamp value) throws HiveException {
+ @Override
+ public Object writeValue(HiveIntervalDayTime value) throws HiveException {
+ throw new HiveException("Internal error: should not reach here");
+ }
+
+ /**
+ * The base implementation must be overridden by the HiveIntervalDayTime specialization
+ */
+ public Object setValue(Object field, HiveIntervalDayTimeWritable value) throws HiveException {
+ throw new HiveException("Internal error: should not reach here");
+ }
+
+ /**
+ * The base implementation must be overridden by the HiveIntervalDayTime specialization
+ */
+ public Object setValue(Object field, HiveIntervalDayTime value) throws HiveException {
throw new HiveException("Internal error: should not reach here");
}
}
@@ -465,6 +486,66 @@ public final class VectorExpressionWriterFactory {
}
}
+ /**
+ * Specialized writer for IntervalDayTimeColumnVector. Will throw cast exception
+ * if the wrong vector column is used.
+ */
+ private static abstract class VectorExpressionWriterIntervalDayTime extends VectorExpressionWriterBase {
+ @Override
+ public Object writeValue(ColumnVector column, int row) throws HiveException {
+ IntervalDayTimeColumnVector dcv = (IntervalDayTimeColumnVector) column;
+ HiveIntervalDayTimeWritable intervalDayTimeWritable = (HiveIntervalDayTimeWritable) dcv.getScratchWritable();
+ if (intervalDayTimeWritable == null) {
+ intervalDayTimeWritable = new HiveIntervalDayTimeWritable();
+ dcv.setScratchWritable(intervalDayTimeWritable);
+ }
+ if (dcv.noNulls && !dcv.isRepeating) {
+ return writeValue(TimestampUtils.intervalDayTimeColumnVectorWritable(dcv, row, intervalDayTimeWritable));
+ } else if (dcv.noNulls && dcv.isRepeating) {
+ return writeValue(TimestampUtils.intervalDayTimeColumnVectorWritable(dcv, 0, intervalDayTimeWritable));
+ } else if (!dcv.noNulls && !dcv.isRepeating && !dcv.isNull[row]) {
+ return writeValue(TimestampUtils.intervalDayTimeColumnVectorWritable(dcv, row, intervalDayTimeWritable));
+ } else if (!dcv.noNulls && dcv.isRepeating && !dcv.isNull[0]) {
+ return writeValue(TimestampUtils.intervalDayTimeColumnVectorWritable(dcv, 0, intervalDayTimeWritable));
+ } else if (!dcv.noNulls && dcv.isRepeating && dcv.isNull[0]) {
+ return null;
+ } else if (!dcv.noNulls && !dcv.isRepeating && dcv.isNull[row]) {
+ return null;
+ }
+ throw new HiveException(
+ String.format(
+ "Incorrect null/repeating: row:%d noNulls:%b isRepeating:%b isNull[row]:%b isNull[0]:%b",
+ row, dcv.noNulls, dcv.isRepeating, dcv.isNull[row], dcv.isNull[0]));
+ }
+
+ @Override
+ public Object setValue(Object field, ColumnVector column, int row) throws HiveException {
+ IntervalDayTimeColumnVector dcv = (IntervalDayTimeColumnVector) column;
+ HiveIntervalDayTimeWritable intervalDayTimeWritable = (HiveIntervalDayTimeWritable) dcv.getScratchWritable();
+ if (intervalDayTimeWritable == null) {
+ intervalDayTimeWritable = new HiveIntervalDayTimeWritable();
+ dcv.setScratchWritable(intervalDayTimeWritable);
+ }
+ if (dcv.noNulls && !dcv.isRepeating) {
+ return setValue(field, TimestampUtils.intervalDayTimeColumnVectorWritable(dcv, row, intervalDayTimeWritable));
+ } else if (dcv.noNulls && dcv.isRepeating) {
+ return setValue(field, TimestampUtils.intervalDayTimeColumnVectorWritable(dcv, 0, intervalDayTimeWritable));
+ } else if (!dcv.noNulls && !dcv.isRepeating && !dcv.isNull[row]) {
+ return setValue(field, TimestampUtils.intervalDayTimeColumnVectorWritable(dcv, row, intervalDayTimeWritable));
+ } else if (!dcv.noNulls && !dcv.isRepeating && dcv.isNull[row]) {
+ return null;
+ } else if (!dcv.noNulls && dcv.isRepeating && !dcv.isNull[0]) {
+ return setValue(field, TimestampUtils.intervalDayTimeColumnVectorWritable(dcv, 0, intervalDayTimeWritable));
+ } else if (!dcv.noNulls && dcv.isRepeating && dcv.isNull[0]) {
+ return null;
+ }
+ throw new HiveException(
+ String.format(
+ "Incorrect null/repeating: row:%d noNulls:%b isRepeating:%b isNull[row]:%b isNull[0]:%b",
+ row, dcv.noNulls, dcv.isRepeating, dcv.isNull[row], dcv.isNull[0]));
+ }
+ }
+
/**
* Compiles the appropriate vector expression writer based on an expression info (ExprNodeDesc)
*/
@@ -697,8 +778,13 @@ public final class VectorExpressionWriterFactory {
}
@Override
- public Object writeValue(PisaTimestamp value) throws HiveException {
- return ((SettableTimestampObjectInspector) this.objectInspector).set(obj, value.asScratchTimestamp());
+ public Object writeValue(HiveIntervalDayTimeWritable value) throws HiveException {
+ return ((SettableHiveIntervalDayTimeObjectInspector) this.objectInspector).set(obj, value);
+ }
+
+ @Override
+ public Object writeValue(HiveIntervalDayTime value) throws HiveException {
+ return ((SettableHiveIntervalDayTimeObjectInspector) this.objectInspector).set(obj, value);
}
@Override
@@ -766,53 +852,45 @@ public final class VectorExpressionWriterFactory {
private static VectorExpressionWriter genVectorExpressionWritableIntervalDayTime(
SettableHiveIntervalDayTimeObjectInspector fieldObjInspector) throws HiveException {
- return new VectorExpressionWriterTimestamp() {
+ return new VectorExpressionWriterIntervalDayTime() {
private Object obj;
private HiveIntervalDayTime interval;
- private PisaTimestamp pisaTimestamp;
public VectorExpressionWriter init(SettableHiveIntervalDayTimeObjectInspector objInspector)
throws HiveException {
super.init(objInspector);
interval = new HiveIntervalDayTime();
obj = initValue(null);
- pisaTimestamp = new PisaTimestamp();
return this;
}
@Override
- public Object writeValue(TimestampWritable value) throws HiveException {
- interval.set(pisaTimestamp.updateFromTimestamp(value.getTimestamp()));
+ public Object writeValue(HiveIntervalDayTimeWritable value) throws HiveException {
+ interval.set(value.getHiveIntervalDayTime());
return ((SettableHiveIntervalDayTimeObjectInspector) this.objectInspector).set(obj, interval);
}
@Override
- public Object writeValue(Timestamp value) throws HiveException {
- interval.set(pisaTimestamp.updateFromTimestamp(value));
- return ((SettableHiveIntervalDayTimeObjectInspector) this.objectInspector).set(obj, interval);
- }
-
- @Override
- public Object writeValue(PisaTimestamp value) throws HiveException {
+ public Object writeValue(HiveIntervalDayTime value) throws HiveException {
interval.set(value);
return ((SettableHiveIntervalDayTimeObjectInspector) this.objectInspector).set(obj, interval);
}
@Override
- public Object setValue(Object field, TimestampWritable value) {
+ public Object setValue(Object field, HiveIntervalDayTimeWritable value) {
if (null == field) {
field = initValue(null);
}
- interval.set(pisaTimestamp.updateFromTimestamp(value.getTimestamp()));
+ interval.set(value.getHiveIntervalDayTime());
return ((SettableHiveIntervalDayTimeObjectInspector) this.objectInspector).set(field, interval);
}
@Override
- public Object setValue(Object field, Timestamp value) {
+ public Object setValue(Object field, HiveIntervalDayTime value) {
if (null == field) {
field = initValue(null);
}
- interval.set(pisaTimestamp.updateFromTimestamp(value));
+ interval.set(value);
return ((SettableHiveIntervalDayTimeObjectInspector) this.objectInspector).set(field, interval);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateAddColCol.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateAddColCol.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateAddColCol.java
index 9f5c793..05dd93e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateAddColCol.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateAddColCol.java
@@ -166,7 +166,7 @@ public class VectorUDFDateAddColCol extends VectorExpression {
protected byte[] evaluateTimestamp(ColumnVector columnVector, int index, long numDays) {
TimestampColumnVector tcv = (TimestampColumnVector) columnVector;
- calendar.setTimeInMillis(tcv.getTimestampMilliseconds(index));
+ calendar.setTimeInMillis(tcv.getTime(index));
if (isPositive) {
calendar.add(Calendar.DATE, (int) numDays);
} else {
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateAddColScalar.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateAddColScalar.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateAddColScalar.java
index 6390ecd..59ca61e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateAddColScalar.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateAddColScalar.java
@@ -210,7 +210,7 @@ public class VectorUDFDateAddColScalar extends VectorExpression {
protected byte[] evaluateTimestamp(ColumnVector columnVector, int index) {
TimestampColumnVector tcv = (TimestampColumnVector) columnVector;
- calendar.setTimeInMillis(tcv.getTimestampMilliseconds(index));
+ calendar.setTimeInMillis(tcv.getTime(index));
if (isPositive) {
calendar.add(Calendar.DATE, numDays);
} else {
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffColCol.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffColCol.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffColCol.java
index b22c31f..4edf558 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffColCol.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffColCol.java
@@ -275,7 +275,7 @@ public class VectorUDFDateDiffColCol extends VectorExpression {
output.isRepeating = true;
if (!input.isNull[0]) {
- date.setTime(input.getTimestampMilliseconds(0));
+ date.setTime(input.getTime(0));
output.vector[0] = DateWritable.dateToDays(date);
}
return;
@@ -288,12 +288,12 @@ public class VectorUDFDateDiffColCol extends VectorExpression {
if (selectedInUse) {
for (int j = 0; j < size; j++) {
int i = sel[j];
- date.setTime(input.getTimestampMilliseconds(i));
+ date.setTime(input.getTime(i));
output.vector[i] = DateWritable.dateToDays(date);
}
} else {
for (int i = 0; i < size; i++) {
- date.setTime(input.getTimestampMilliseconds(i));
+ date.setTime(input.getTime(i));
output.vector[i] = DateWritable.dateToDays(date);
}
}
@@ -312,14 +312,14 @@ public class VectorUDFDateDiffColCol extends VectorExpression {
for (int j = 0; j < size; j++) {
int i = sel[j];
if (!input.isNull[i]) {
- date.setTime(input.getTimestampMilliseconds(i));
+ date.setTime(input.getTime(i));
output.vector[i] = DateWritable.dateToDays(date);
}
}
} else {
for (int i = 0; i < size; i++) {
if (!input.isNull[i]) {
- date.setTime(input.getTimestampMilliseconds(i));
+ date.setTime(input.getTime(i));
output.vector[i] = DateWritable.dateToDays(date);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffColScalar.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffColScalar.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffColScalar.java
index ab71b47..71b3887 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffColScalar.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffColScalar.java
@@ -238,7 +238,7 @@ public class VectorUDFDateDiffColScalar extends VectorExpression {
protected int evaluateTimestamp(ColumnVector columnVector, int index) {
TimestampColumnVector tcv = (TimestampColumnVector) columnVector;
- date.setTime(tcv.getTimestampMilliseconds(index));
+ date.setTime(tcv.getTime(index));
return DateWritable.dateToDays(date) - baseDate;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffScalarCol.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffScalarCol.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffScalarCol.java
index dea5444..c733bc9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffScalarCol.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffScalarCol.java
@@ -237,7 +237,7 @@ public class VectorUDFDateDiffScalarCol extends VectorExpression {
protected int evaluateTimestamp(ColumnVector columnVector, int index) {
TimestampColumnVector tcv = (TimestampColumnVector) columnVector;
- date.setTime(tcv.getTimestampMilliseconds(index));
+ date.setTime(tcv.getTime(index));
return baseDate - DateWritable.dateToDays(date);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateTimestamp.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateTimestamp.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateTimestamp.java
index c29e22e..cde0be4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateTimestamp.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateTimestamp.java
@@ -45,7 +45,7 @@ public class VectorUDFDateTimestamp extends TimestampToStringUnaryUDF {
protected void func(BytesColumnVector outV, TimestampColumnVector inV, int i) {
switch (inputTypes[0]) {
case TIMESTAMP:
- date.setTime(inV.getTimestampMilliseconds(i));
+ date.setTime(inV.getTime(i));
break;
default:
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFUnixTimeStampDate.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFUnixTimeStampDate.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFUnixTimeStampDate.java
index b7c4ff4..3c693af 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFUnixTimeStampDate.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFUnixTimeStampDate.java
@@ -28,15 +28,18 @@ public final class VectorUDFUnixTimeStampDate extends VectorUDFTimestampFieldDat
private static final long serialVersionUID = 1L;
+ private DateWritable dateWritable;
+
@Override
protected long getDateField(long days) {
- long ms = DateWritable.daysToMillis((int) days);
- return ms / 1000;
+ dateWritable.set((int) days);
+ return dateWritable.getTimeInSeconds();
}
public VectorUDFUnixTimeStampDate(int colNum, int outputColumn) {
/* not a real field */
super(-1, colNum, outputColumn);
+ dateWritable = new DateWritable();
}
public VectorUDFUnixTimeStampDate() {
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFUnixTimeStampTimestamp.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFUnixTimeStampTimestamp.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFUnixTimeStampTimestamp.java
index e4a31ca..2bd7756 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFUnixTimeStampTimestamp.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFUnixTimeStampTimestamp.java
@@ -18,10 +18,7 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions;
-import java.sql.Timestamp;
-
import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
-import org.apache.hadoop.hive.serde2.io.DateWritable;
/**
* Return Unix Timestamp.
@@ -33,7 +30,7 @@ public final class VectorUDFUnixTimeStampTimestamp extends VectorUDFTimestampFie
@Override
protected long getTimestampField(TimestampColumnVector timestampColVector, int elementNum) {
- return timestampColVector.getTimestampSeconds(elementNum);
+ return timestampColVector.asScratchTimestamp(elementNum).getTime() / 1000;
}
public VectorUDFUnixTimeStampTimestamp(int colNum, int outputColumn) {
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFAvgTimestamp.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFAvgTimestamp.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFAvgTimestamp.java
index 5c8db41..d0a1d0d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFAvgTimestamp.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFAvgTimestamp.java
@@ -27,8 +27,6 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggreg
import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.VectorAggregationBufferRow;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
-import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.AggregationDesc;
import org.apache.hadoop.hive.ql.util.JavaDataModel;
@@ -146,7 +144,8 @@ public class VectorUDAFAvgTimestamp extends VectorAggregateExpression {
if (inputColVector.isRepeating) {
iterateNoNullsRepeatingWithAggregationSelection(
aggregationBufferSets, bufferIndex,
- inputColVector.getTimestampSecondsWithFractionalNanos(0), batchSize);
+ inputColVector.getDouble(0),
+ batchSize);
} else {
if (batch.selectedInUse) {
iterateNoNullsSelectionWithAggregationSelection(
@@ -163,11 +162,11 @@ public class VectorUDAFAvgTimestamp extends VectorAggregateExpression {
if (batch.selectedInUse) {
iterateHasNullsRepeatingSelectionWithAggregationSelection(
aggregationBufferSets, bufferIndex,
- inputColVector.getTimestampSecondsWithFractionalNanos(0), batchSize, batch.selected, inputColVector.isNull);
+ inputColVector.getDouble(0), batchSize, batch.selected, inputColVector.isNull);
} else {
iterateHasNullsRepeatingWithAggregationSelection(
aggregationBufferSets, bufferIndex,
- inputColVector.getTimestampSecondsWithFractionalNanos(0), batchSize, inputColVector.isNull);
+ inputColVector.getDouble(0), batchSize, inputColVector.isNull);
}
} else {
if (batch.selectedInUse) {
@@ -210,7 +209,8 @@ public class VectorUDAFAvgTimestamp extends VectorAggregateExpression {
aggregationBufferSets,
bufferIndex,
i);
- myagg.sumValue(inputColVector.getTimestampSecondsWithFractionalNanos(selection[i]));
+ myagg.sumValue(
+ inputColVector.getDouble(selection[i]));
}
}
@@ -224,7 +224,7 @@ public class VectorUDAFAvgTimestamp extends VectorAggregateExpression {
aggregationBufferSets,
bufferIndex,
i);
- myagg.sumValue(inputColVector.getTimestampSecondsWithFractionalNanos(i));
+ myagg.sumValue(inputColVector.getDouble(i));
}
}
@@ -281,7 +281,7 @@ public class VectorUDAFAvgTimestamp extends VectorAggregateExpression {
aggregationBufferSets,
bufferIndex,
j);
- myagg.sumValue(inputColVector.getTimestampSecondsWithFractionalNanos(i));
+ myagg.sumValue(inputColVector.getDouble(i));
}
}
}
@@ -296,10 +296,10 @@ public class VectorUDAFAvgTimestamp extends VectorAggregateExpression {
for (int i=0; i < batchSize; ++i) {
if (!isNull[i]) {
Aggregation myagg = getCurrentAggregationBuffer(
- aggregationBufferSets,
+ aggregationBufferSets,
bufferIndex,
i);
- myagg.sumValue(inputColVector.getTimestampSecondsWithFractionalNanos(i));
+ myagg.sumValue(inputColVector.getDouble(i));
}
}
}
@@ -328,7 +328,7 @@ public class VectorUDAFAvgTimestamp extends VectorAggregateExpression {
myagg.sum = 0;
myagg.count = 0;
}
- myagg.sum += inputColVector.getTimestampSecondsWithFractionalNanos(0)*batchSize;
+ myagg.sum += inputColVector.getDouble(0)*batchSize;
myagg.count += batchSize;
}
return;
@@ -358,7 +358,7 @@ public class VectorUDAFAvgTimestamp extends VectorAggregateExpression {
for (int j=0; j< batchSize; ++j) {
int i = selected[j];
if (!isNull[i]) {
- double value = inputColVector.getTimestampSecondsWithFractionalNanos(i);
+ double value = inputColVector.getDouble(i);
if (myagg.isNull) {
myagg.isNull = false;
myagg.sum = 0;
@@ -381,24 +381,24 @@ public class VectorUDAFAvgTimestamp extends VectorAggregateExpression {
myagg.sum = 0;
myagg.count = 0;
}
-
+
for (int i=0; i< batchSize; ++i) {
- double value = inputColVector.getTimestampSecondsWithFractionalNanos(selected[i]);
+ double value = inputColVector.getDouble(selected[i]);
myagg.sum += value;
myagg.count += 1;
}
}
private void iterateNoSelectionHasNulls(
- Aggregation myagg,
- TimestampColumnVector inputColVector,
+ Aggregation myagg,
+ TimestampColumnVector inputColVector,
int batchSize,
boolean[] isNull) {
-
+
for(int i=0;i<batchSize;++i) {
if (!isNull[i]) {
- double value = inputColVector.getTimestampSecondsWithFractionalNanos(i);
- if (myagg.isNull) {
+ double value = inputColVector.getDouble(i);
+ if (myagg.isNull) {
myagg.isNull = false;
myagg.sum = 0;
myagg.count = 0;
@@ -420,7 +420,7 @@ public class VectorUDAFAvgTimestamp extends VectorAggregateExpression {
}
for (int i=0;i<batchSize;++i) {
- double value = inputColVector.getTimestampSecondsWithFractionalNanos(i);
+ double value = inputColVector.getDouble(i);
myagg.sum += value;
myagg.count += 1;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFStdPopTimestamp.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFStdPopTimestamp.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFStdPopTimestamp.java
index 17906ec..fa25e6a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFStdPopTimestamp.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFStdPopTimestamp.java
@@ -152,7 +152,7 @@ public class VectorUDAFStdPopTimestamp extends VectorAggregateExpression {
if (inputColVector.isRepeating) {
if (inputColVector.noNulls || !inputColVector.isNull[0]) {
iterateRepeatingNoNullsWithAggregationSelection(
- aggregationBufferSets, aggregateIndex, inputColVector.getTimestampSecondsWithFractionalNanos(0), batchSize);
+ aggregationBufferSets, aggregateIndex, inputColVector.getDouble(0), batchSize);
}
}
else if (!batch.selectedInUse && inputColVector.noNulls) {
@@ -213,7 +213,7 @@ public class VectorUDAFStdPopTimestamp extends VectorAggregateExpression {
j);
int i = selected[j];
if (!isNull[i]) {
- double value = inputColVector.getTimestampSecondsWithFractionalNanos(i);
+ double value = inputColVector.getDouble(i);
if (myagg.isNull) {
myagg.init ();
}
@@ -239,7 +239,7 @@ public class VectorUDAFStdPopTimestamp extends VectorAggregateExpression {
aggregationBufferSets,
aggregateIndex,
i);
- double value = inputColVector.getTimestampSecondsWithFractionalNanos(selected[i]);
+ double value = inputColVector.getDouble(selected[i]);
if (myagg.isNull) {
myagg.init ();
}
@@ -265,7 +265,7 @@ public class VectorUDAFStdPopTimestamp extends VectorAggregateExpression {
aggregationBufferSets,
aggregateIndex,
i);
- double value = inputColVector.getTimestampSecondsWithFractionalNanos(i);
+ double value = inputColVector.getDouble(i);
if (myagg.isNull) {
myagg.init ();
}
@@ -293,7 +293,7 @@ public class VectorUDAFStdPopTimestamp extends VectorAggregateExpression {
if (myagg.isNull) {
myagg.init ();
}
- double value = inputColVector.getTimestampSecondsWithFractionalNanos(i);
+ double value = inputColVector.getDouble(i);
myagg.sum += value;
myagg.count += 1;
if(myagg.count > 1) {
@@ -322,7 +322,7 @@ public class VectorUDAFStdPopTimestamp extends VectorAggregateExpression {
if (inputColVector.isRepeating) {
if (inputColVector.noNulls) {
- iterateRepeatingNoNulls(myagg, inputColVector.getTimestampSecondsWithFractionalNanos(0), batchSize);
+ iterateRepeatingNoNulls(myagg, inputColVector.getDouble(0), batchSize);
}
}
else if (!batch.selectedInUse && inputColVector.noNulls) {
@@ -377,7 +377,7 @@ public class VectorUDAFStdPopTimestamp extends VectorAggregateExpression {
for (int j=0; j< batchSize; ++j) {
int i = selected[j];
if (!isNull[i]) {
- double value = inputColVector.getTimestampSecondsWithFractionalNanos(i);
+ double value = inputColVector.getDouble(i);
if (myagg.isNull) {
myagg.init ();
}
@@ -401,7 +401,7 @@ public class VectorUDAFStdPopTimestamp extends VectorAggregateExpression {
myagg.init ();
}
- double value = inputColVector.getTimestampSecondsWithFractionalNanos(selected[0]);
+ double value = inputColVector.getDouble(selected[0]);
myagg.sum += value;
myagg.count += 1;
if(myagg.count > 1) {
@@ -412,7 +412,7 @@ public class VectorUDAFStdPopTimestamp extends VectorAggregateExpression {
// i=0 was pulled out to remove the count > 1 check in the loop
//
for (int i=1; i< batchSize; ++i) {
- value = inputColVector.getTimestampSecondsWithFractionalNanos(selected[i]);
+ value = inputColVector.getDouble(selected[i]);
myagg.sum += value;
myagg.count += 1;
double t = myagg.count*value - myagg.sum;
@@ -428,7 +428,7 @@ public class VectorUDAFStdPopTimestamp extends VectorAggregateExpression {
for(int i=0;i<batchSize;++i) {
if (!isNull[i]) {
- double value = inputColVector.getTimestampSecondsWithFractionalNanos(i);
+ double value = inputColVector.getDouble(i);
if (myagg.isNull) {
myagg.init ();
}
@@ -451,7 +451,7 @@ public class VectorUDAFStdPopTimestamp extends VectorAggregateExpression {
myagg.init ();
}
- double value = inputColVector.getTimestampSecondsWithFractionalNanos(0);
+ double value = inputColVector.getDouble(0);
myagg.sum += value;
myagg.count += 1;
@@ -462,7 +462,7 @@ public class VectorUDAFStdPopTimestamp extends VectorAggregateExpression {
// i=0 was pulled out to remove count > 1 check
for (int i=1; i<batchSize; ++i) {
- value = inputColVector.getTimestampSecondsWithFractionalNanos(i);
+ value = inputColVector.getDouble(i);
myagg.sum += value;
myagg.count += 1;
double t = myagg.count*value - myagg.sum;
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFStdSampTimestamp.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFStdSampTimestamp.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFStdSampTimestamp.java
index 2e41e47..b3e1fae 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFStdSampTimestamp.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFStdSampTimestamp.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression;
import org.apache.hadoop.hive.ql.exec.vector.VectorAggregationBufferRow;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
-import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.AggregationDesc;
@@ -38,7 +37,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
/**
-* VectorUDAFStdSampDouble. Vectorized implementation for VARIANCE aggregates.
+* VectorUDAFStdSampTimestamp. Vectorized implementation for VARIANCE aggregates.
*/
@Description(name = "stddev_samp",
value = "_FUNC_(x) - Returns the sample standard deviation of a set of numbers (vectorized, double)")
@@ -153,7 +152,7 @@ public class VectorUDAFStdSampTimestamp extends VectorAggregateExpression {
if (inputColVector.isRepeating) {
if (inputColVector.noNulls || !inputColVector.isNull[0]) {
iterateRepeatingNoNullsWithAggregationSelection(
- aggregationBufferSets, aggregateIndex, inputColVector.getTimestampSecondsWithFractionalNanos(0), batchSize);
+ aggregationBufferSets, aggregateIndex, inputColVector.getDouble(0), batchSize);
}
}
else if (!batch.selectedInUse && inputColVector.noNulls) {
@@ -214,7 +213,7 @@ public class VectorUDAFStdSampTimestamp extends VectorAggregateExpression {
j);
int i = selected[j];
if (!isNull[i]) {
- double value = inputColVector.getTimestampSecondsWithFractionalNanos(i);
+ double value = inputColVector.getDouble(i);
if (myagg.isNull) {
myagg.init ();
}
@@ -240,7 +239,7 @@ public class VectorUDAFStdSampTimestamp extends VectorAggregateExpression {
aggregationBufferSets,
aggregateIndex,
i);
- double value = inputColVector.getTimestampSecondsWithFractionalNanos(selected[i]);
+ double value = inputColVector.getDouble(selected[i]);
if (myagg.isNull) {
myagg.init ();
}
@@ -266,7 +265,7 @@ public class VectorUDAFStdSampTimestamp extends VectorAggregateExpression {
aggregationBufferSets,
aggregateIndex,
i);
- double value = inputColVector.getTimestampSecondsWithFractionalNanos(i);
+ double value = inputColVector.getDouble(i);
if (myagg.isNull) {
myagg.init ();
}
@@ -294,7 +293,7 @@ public class VectorUDAFStdSampTimestamp extends VectorAggregateExpression {
if (myagg.isNull) {
myagg.init ();
}
- double value = inputColVector.getTimestampSecondsWithFractionalNanos(i);
+ double value = inputColVector.getDouble(i);
myagg.sum += value;
myagg.count += 1;
if(myagg.count > 1) {
@@ -323,7 +322,7 @@ public class VectorUDAFStdSampTimestamp extends VectorAggregateExpression {
if (inputColVector.isRepeating) {
if (inputColVector.noNulls) {
- iterateRepeatingNoNulls(myagg, inputColVector.getTimestampSecondsWithFractionalNanos(0), batchSize);
+ iterateRepeatingNoNulls(myagg, inputColVector.getDouble(0), batchSize);
}
}
else if (!batch.selectedInUse && inputColVector.noNulls) {
@@ -378,7 +377,7 @@ public class VectorUDAFStdSampTimestamp extends VectorAggregateExpression {
for (int j=0; j< batchSize; ++j) {
int i = selected[j];
if (!isNull[i]) {
- double value = inputColVector.getTimestampSecondsWithFractionalNanos(i);
+ double value = inputColVector.getDouble(i);
if (myagg.isNull) {
myagg.init ();
}
@@ -402,7 +401,7 @@ public class VectorUDAFStdSampTimestamp extends VectorAggregateExpression {
myagg.init ();
}
- double value = inputColVector.getTimestampSecondsWithFractionalNanos(selected[0]);
+ double value = inputColVector.getDouble(selected[0]);
myagg.sum += value;
myagg.count += 1;
if(myagg.count > 1) {
@@ -413,7 +412,7 @@ public class VectorUDAFStdSampTimestamp extends VectorAggregateExpression {
// i=0 was pulled out to remove the count > 1 check in the loop
//
for (int i=1; i< batchSize; ++i) {
- value = inputColVector.getTimestampSecondsWithFractionalNanos(selected[i]);
+ value = inputColVector.getDouble(selected[i]);
myagg.sum += value;
myagg.count += 1;
double t = myagg.count*value - myagg.sum;
@@ -429,7 +428,7 @@ public class VectorUDAFStdSampTimestamp extends VectorAggregateExpression {
for(int i=0;i<batchSize;++i) {
if (!isNull[i]) {
- double value = inputColVector.getTimestampSecondsWithFractionalNanos(i);
+ double value = inputColVector.getDouble(i);
if (myagg.isNull) {
myagg.init ();
}
@@ -452,7 +451,7 @@ public class VectorUDAFStdSampTimestamp extends VectorAggregateExpression {
myagg.init ();
}
- double value = inputColVector.getTimestampSecondsWithFractionalNanos(0);
+ double value = inputColVector.getDouble(0);
myagg.sum += value;
myagg.count += 1;
@@ -463,7 +462,7 @@ public class VectorUDAFStdSampTimestamp extends VectorAggregateExpression {
// i=0 was pulled out to remove count > 1 check
for (int i=1; i<batchSize; ++i) {
- value = inputColVector.getTimestampSecondsWithFractionalNanos(i);
+ value = inputColVector.getDouble(i);
myagg.sum += value;
myagg.count += 1;
double t = myagg.count*value - myagg.sum;
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFVarPopTimestamp.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFVarPopTimestamp.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFVarPopTimestamp.java
index d128b7c..970ec22 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFVarPopTimestamp.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFVarPopTimestamp.java
@@ -152,7 +152,7 @@ public class VectorUDAFVarPopTimestamp extends VectorAggregateExpression {
if (inputColVector.isRepeating) {
if (inputColVector.noNulls || !inputColVector.isNull[0]) {
iterateRepeatingNoNullsWithAggregationSelection(
- aggregationBufferSets, aggregateIndex, inputColVector.getTimestampSecondsWithFractionalNanos(0), batchSize);
+ aggregationBufferSets, aggregateIndex, inputColVector.getDouble(0), batchSize);
}
}
else if (!batch.selectedInUse && inputColVector.noNulls) {
@@ -213,7 +213,7 @@ public class VectorUDAFVarPopTimestamp extends VectorAggregateExpression {
j);
int i = selected[j];
if (!isNull[i]) {
- double value = inputColVector.getTimestampSecondsWithFractionalNanos(i);
+ double value = inputColVector.getDouble(i);
if (myagg.isNull) {
myagg.init ();
}
@@ -239,7 +239,7 @@ public class VectorUDAFVarPopTimestamp extends VectorAggregateExpression {
aggregationBufferSets,
aggregateIndex,
i);
- double value = inputColVector.getTimestampSecondsWithFractionalNanos(selected[i]);
+ double value = inputColVector.getDouble(selected[i]);
if (myagg.isNull) {
myagg.init ();
}
@@ -265,7 +265,7 @@ public class VectorUDAFVarPopTimestamp extends VectorAggregateExpression {
aggregationBufferSets,
aggregateIndex,
i);
- double value = inputColVector.getTimestampSecondsWithFractionalNanos(i);
+ double value = inputColVector.getDouble(i);
if (myagg.isNull) {
myagg.init ();
}
@@ -293,7 +293,7 @@ public class VectorUDAFVarPopTimestamp extends VectorAggregateExpression {
if (myagg.isNull) {
myagg.init ();
}
- double value = inputColVector.getTimestampSecondsWithFractionalNanos(i);
+ double value = inputColVector.getDouble(i);
myagg.sum += value;
myagg.count += 1;
if(myagg.count > 1) {
@@ -322,7 +322,7 @@ public class VectorUDAFVarPopTimestamp extends VectorAggregateExpression {
if (inputColVector.isRepeating) {
if (inputColVector.noNulls) {
- iterateRepeatingNoNulls(myagg, inputColVector.getTimestampSecondsWithFractionalNanos(0), batchSize);
+ iterateRepeatingNoNulls(myagg, inputColVector.getDouble(0), batchSize);
}
}
else if (!batch.selectedInUse && inputColVector.noNulls) {
@@ -377,7 +377,7 @@ public class VectorUDAFVarPopTimestamp extends VectorAggregateExpression {
for (int j=0; j< batchSize; ++j) {
int i = selected[j];
if (!isNull[i]) {
- double value = inputColVector.getTimestampSecondsWithFractionalNanos(i);
+ double value = inputColVector.getDouble(i);
if (myagg.isNull) {
myagg.init ();
}
@@ -401,7 +401,7 @@ public class VectorUDAFVarPopTimestamp extends VectorAggregateExpression {
myagg.init ();
}
- double value = inputColVector.getTimestampSecondsWithFractionalNanos(selected[0]);
+ double value = inputColVector.getDouble(selected[0]);
myagg.sum += value;
myagg.count += 1;
if(myagg.count > 1) {
@@ -412,7 +412,7 @@ public class VectorUDAFVarPopTimestamp extends VectorAggregateExpression {
// i=0 was pulled out to remove the count > 1 check in the loop
//
for (int i=1; i< batchSize; ++i) {
- value = inputColVector.getTimestampSecondsWithFractionalNanos(selected[i]);
+ value = inputColVector.getDouble(selected[i]);
myagg.sum += value;
myagg.count += 1;
double t = myagg.count*value - myagg.sum;
@@ -428,7 +428,7 @@ public class VectorUDAFVarPopTimestamp extends VectorAggregateExpression {
for(int i=0;i<batchSize;++i) {
if (!isNull[i]) {
- double value = inputColVector.getTimestampSecondsWithFractionalNanos(i);
+ double value = inputColVector.getDouble(i);
if (myagg.isNull) {
myagg.init ();
}
@@ -451,7 +451,7 @@ public class VectorUDAFVarPopTimestamp extends VectorAggregateExpression {
myagg.init ();
}
- double value = inputColVector.getTimestampSecondsWithFractionalNanos(0);
+ double value = inputColVector.getDouble(0);
myagg.sum += value;
myagg.count += 1;
@@ -462,7 +462,7 @@ public class VectorUDAFVarPopTimestamp extends VectorAggregateExpression {
// i=0 was pulled out to remove count > 1 check
for (int i=1; i<batchSize; ++i) {
- value = inputColVector.getTimestampSecondsWithFractionalNanos(i);
+ value = inputColVector.getDouble(i);
myagg.sum += value;
myagg.count += 1;
double t = myagg.count*value - myagg.sum;
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFVarSampTimestamp.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFVarSampTimestamp.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFVarSampTimestamp.java
index cf76f20..9af1a28 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFVarSampTimestamp.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFVarSampTimestamp.java
@@ -152,7 +152,7 @@ public class VectorUDAFVarSampTimestamp extends VectorAggregateExpression {
if (inputColVector.isRepeating) {
if (inputColVector.noNulls || !inputColVector.isNull[0]) {
iterateRepeatingNoNullsWithAggregationSelection(
- aggregationBufferSets, aggregateIndex, inputColVector.getTimestampSecondsWithFractionalNanos(0), batchSize);
+ aggregationBufferSets, aggregateIndex, inputColVector.getDouble(0), batchSize);
}
}
else if (!batch.selectedInUse && inputColVector.noNulls) {
@@ -213,7 +213,7 @@ public class VectorUDAFVarSampTimestamp extends VectorAggregateExpression {
j);
int i = selected[j];
if (!isNull[i]) {
- double value = inputColVector.getTimestampSecondsWithFractionalNanos(i);
+ double value = inputColVector.getDouble(i);
if (myagg.isNull) {
myagg.init ();
}
@@ -239,7 +239,7 @@ public class VectorUDAFVarSampTimestamp extends VectorAggregateExpression {
aggregationBufferSets,
aggregateIndex,
i);
- double value = inputColVector.getTimestampSecondsWithFractionalNanos(selected[i]);
+ double value = inputColVector.getDouble(selected[i]);
if (myagg.isNull) {
myagg.init ();
}
@@ -265,7 +265,7 @@ public class VectorUDAFVarSampTimestamp extends VectorAggregateExpression {
aggregationBufferSets,
aggregateIndex,
i);
- double value = inputColVector.getTimestampSecondsWithFractionalNanos(i);
+ double value = inputColVector.getDouble(i);
if (myagg.isNull) {
myagg.init ();
}
@@ -293,7 +293,7 @@ public class VectorUDAFVarSampTimestamp extends VectorAggregateExpression {
if (myagg.isNull) {
myagg.init ();
}
- double value = inputColVector.getTimestampSecondsWithFractionalNanos(i);
+ double value = inputColVector.getDouble(i);
myagg.sum += value;
myagg.count += 1;
if(myagg.count > 1) {
@@ -322,7 +322,7 @@ public class VectorUDAFVarSampTimestamp extends VectorAggregateExpression {
if (inputColVector.isRepeating) {
if (inputColVector.noNulls) {
- iterateRepeatingNoNulls(myagg, inputColVector.getTimestampSecondsWithFractionalNanos(0), batchSize);
+ iterateRepeatingNoNulls(myagg, inputColVector.getDouble(0), batchSize);
}
}
else if (!batch.selectedInUse && inputColVector.noNulls) {
@@ -377,7 +377,7 @@ public class VectorUDAFVarSampTimestamp extends VectorAggregateExpression {
for (int j=0; j< batchSize; ++j) {
int i = selected[j];
if (!isNull[i]) {
- double value = inputColVector.getTimestampSecondsWithFractionalNanos(i);
+ double value = inputColVector.getDouble(i);
if (myagg.isNull) {
myagg.init ();
}
@@ -401,7 +401,7 @@ public class VectorUDAFVarSampTimestamp extends VectorAggregateExpression {
myagg.init ();
}
- double value = inputColVector.getTimestampSecondsWithFractionalNanos(selected[0]);
+ double value = inputColVector.getDouble(selected[0]);
myagg.sum += value;
myagg.count += 1;
if(myagg.count > 1) {
@@ -412,7 +412,7 @@ public class VectorUDAFVarSampTimestamp extends VectorAggregateExpression {
// i=0 was pulled out to remove the count > 1 check in the loop
//
for (int i=1; i< batchSize; ++i) {
- value = inputColVector.getTimestampSecondsWithFractionalNanos(selected[i]);
+ value = inputColVector.getDouble(selected[i]);
myagg.sum += value;
myagg.count += 1;
double t = myagg.count*value - myagg.sum;
@@ -428,7 +428,7 @@ public class VectorUDAFVarSampTimestamp extends VectorAggregateExpression {
for(int i=0;i<batchSize;++i) {
if (!isNull[i]) {
- double value = inputColVector.getTimestampSecondsWithFractionalNanos(i);
+ double value = inputColVector.getDouble(i);
if (myagg.isNull) {
myagg.init ();
}
@@ -451,7 +451,7 @@ public class VectorUDAFVarSampTimestamp extends VectorAggregateExpression {
myagg.init ();
}
- double value = inputColVector.getTimestampSecondsWithFractionalNanos(0);
+ double value = inputColVector.getDouble(0);
myagg.sum += value;
myagg.count += 1;
@@ -462,7 +462,7 @@ public class VectorUDAFVarSampTimestamp extends VectorAggregateExpression {
// i=0 was pulled out to remove count > 1 check
for (int i=1; i<batchSize; ++i) {
- value = inputColVector.getTimestampSecondsWithFractionalNanos(i);
+ value = inputColVector.getDouble(i);
myagg.sum += value;
myagg.count += 1;
double t = myagg.count*value - myagg.sum;
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/udf/VectorUDFAdaptor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/udf/VectorUDFAdaptor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/udf/VectorUDFAdaptor.java
index d3a0f9f..20cfb89 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/udf/VectorUDFAdaptor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/udf/VectorUDFAdaptor.java
@@ -297,23 +297,14 @@ public class VectorUDFAdaptor extends VectorExpression {
lv.vector[i] = ((WritableByteObjectInspector) outputOI).get(value);
}
} else if (outputOI instanceof WritableTimestampObjectInspector) {
- LongColumnVector lv = (LongColumnVector) colVec;
+ TimestampColumnVector tv = (TimestampColumnVector) colVec;
Timestamp ts;
if (value instanceof Timestamp) {
ts = (Timestamp) value;
} else {
ts = ((WritableTimestampObjectInspector) outputOI).getPrimitiveJavaObject(value);
}
- /* Calculate the number of nanoseconds since the epoch as a long integer. By convention
- * that is how Timestamp values are operated on in a vector.
- */
- long l = ts.getTime() * 1000000 // Shift the milliseconds value over by 6 digits
- // to scale for nanosecond precision.
- // The milliseconds digits will by convention be all 0s.
- + ts.getNanos() % 1000000; // Add on the remaining nanos.
- // The % 1000000 operation removes the ms values
- // so that the milliseconds are not counted twice.
- lv.vector[i] = l;
+ tv.set(i, ts);
} else if (outputOI instanceof WritableDateObjectInspector) {
LongColumnVector lv = (LongColumnVector) colVec;
Date ts;
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/util/DateTimeMath.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/util/DateTimeMath.java b/ql/src/java/org/apache/hadoop/hive/ql/util/DateTimeMath.java
index e092ac2..98b1ded 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/util/DateTimeMath.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/util/DateTimeMath.java
@@ -25,7 +25,6 @@ import java.util.concurrent.TimeUnit;
import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
import org.apache.hadoop.hive.serde2.io.DateWritable;
import org.apache.hive.common.util.DateUtils;
@@ -91,26 +90,6 @@ public class DateTimeMath {
return result;
}
- /**
- * Perform month arithmetic to millis value using local time zone.
- * @param pisaTimestamp
- * @param months
- * @return
- */
- public PisaTimestamp addMonthsToPisaTimestamp(PisaTimestamp pisaTimestamp, int months,
- PisaTimestamp scratchPisaTimestamp) {
- calLocal.setTimeInMillis(pisaTimestamp.getTimestampMilliseconds());
- calLocal.add(Calendar.MONTH, months);
- scratchPisaTimestamp.updateFromTimestampMilliseconds(calLocal.getTimeInMillis());
-
- // Add in portion of nanos below a millisecond...
- PisaTimestamp.add(
- scratchPisaTimestamp.getEpochDay(), scratchPisaTimestamp.getNanoOfDay(),
- 0, pisaTimestamp.getNanoOfDay() % 1000000,
- scratchPisaTimestamp);
- return scratchPisaTimestamp;
- }
-
public long addMonthsToDays(long days, int months) {
long millis = DateWritable.daysToMillis((int) days);
millis = addMonthsToMillisLocal(millis, months);
@@ -123,24 +102,95 @@ public class DateTimeMath {
return null;
}
+ Timestamp tsResult = new Timestamp(0);
+ add(ts, interval, tsResult);
+
+ return tsResult;
+ }
+
+ public boolean add(Timestamp ts, HiveIntervalYearMonth interval, Timestamp result) {
+ if (ts == null || interval == null) {
+ return false;
+ }
+
// Attempt to match Oracle semantics for timestamp arithmetic,
// where timestamp arithmetic is done in UTC, then converted back to local timezone
long resultMillis = addMonthsToMillisUtc(ts.getTime(), interval.getTotalMonths());
- Timestamp tsResult = new Timestamp(resultMillis);
- tsResult.setNanos(ts.getNanos());
+ result.setTime(resultMillis);
+ result.setNanos(ts.getNanos());
+
+ return true;
+ }
+
+ public Timestamp add(HiveIntervalYearMonth interval, Timestamp ts) {
+ if (ts == null || interval == null) {
+ return null;
+ }
+
+ Timestamp tsResult = new Timestamp(0);
+ add(interval, ts, tsResult);
return tsResult;
}
+ public boolean add(HiveIntervalYearMonth interval, Timestamp ts, Timestamp result) {
+ if (ts == null || interval == null) {
+ return false;
+ }
+
+ // Attempt to match Oracle semantics for timestamp arithmetic,
+ // where timestamp arithmetic is done in UTC, then converted back to local timezone
+ long resultMillis = addMonthsToMillisUtc(ts.getTime(), interval.getTotalMonths());
+ result.setTime(resultMillis);
+ result.setNanos(ts.getNanos());
+
+ return true;
+ }
+
public Date add(Date dt, HiveIntervalYearMonth interval) {
if (dt == null || interval == null) {
return null;
}
+ Date dtResult = new Date(0);
+ add(dt, interval, dtResult);
+
+ return dtResult;
+ }
+
+ public boolean add(Date dt, HiveIntervalYearMonth interval, Date result) {
+ if (dt == null || interval == null) {
+ return false;
+ }
+
+ // Since Date millis value is in local timezone representation, do date arithmetic
+ // using local timezone so the time remains at the start of the day.
+ long resultMillis = addMonthsToMillisLocal(dt.getTime(), interval.getTotalMonths());
+ result.setTime(resultMillis);
+ return true;
+ }
+
+ public Date add(HiveIntervalYearMonth interval, Date dt) {
+ if (dt == null || interval == null) {
+ return null;
+ }
+
+ Date dtResult = new Date(0);
+ add(interval, dt, dtResult);
+
+ return dtResult;
+ }
+
+ public boolean add(HiveIntervalYearMonth interval, Date dt, Date result) {
+ if (dt == null || interval == null) {
+ return false;
+ }
+
// Since Date millis value is in local timezone representation, do date arithmetic
// using local timezone so the time remains at the start of the day.
long resultMillis = addMonthsToMillisLocal(dt.getTime(), interval.getTotalMonths());
- return new Date(resultMillis);
+ result.setTime(resultMillis);
+ return true;
}
public HiveIntervalYearMonth add(HiveIntervalYearMonth left, HiveIntervalYearMonth right) {
@@ -157,14 +207,36 @@ public class DateTimeMath {
if (left == null || right == null) {
return null;
}
- return add(left, right.negate());
+
+ Timestamp tsResult = new Timestamp(0);
+ subtract(left, right, tsResult);
+
+ return tsResult;
+ }
+
+ public boolean subtract(Timestamp left, HiveIntervalYearMonth right, Timestamp result) {
+ if (left == null || right == null) {
+ return false;
+ }
+ return add(left, right.negate(), result);
}
public Date subtract(Date left, HiveIntervalYearMonth right) {
if (left == null || right == null) {
return null;
}
- return add(left, right.negate());
+
+ Date dtResult = new Date(0);
+ subtract(left, right, dtResult);
+
+ return dtResult;
+ }
+
+ public boolean subtract(Date left, HiveIntervalYearMonth right, Date result) {
+ if (left == null || right == null) {
+ return false;
+ }
+ return add(left, right.negate(), result);
}
public HiveIntervalYearMonth subtract(HiveIntervalYearMonth left, HiveIntervalYearMonth right) {
@@ -183,26 +255,74 @@ public class DateTimeMath {
return null;
}
+ Timestamp tsResult = new Timestamp(0);
+ add(ts, interval, tsResult);
+
+ return tsResult;
+ }
+
+ public boolean add(Timestamp ts, HiveIntervalDayTime interval,
+ Timestamp result) {
+ if (ts == null || interval == null) {
+ return false;
+ }
+
nanosResult.addNanos(ts.getNanos(), interval.getNanos());
long newMillis = ts.getTime()
+ TimeUnit.SECONDS.toMillis(interval.getTotalSeconds() + nanosResult.seconds);
- Timestamp tsResult = new Timestamp(newMillis);
- tsResult.setNanos(nanosResult.nanos);
+ result.setTime(newMillis);
+ result.setNanos(nanosResult.nanos);
+ return true;
+ }
+
+ public Timestamp add(HiveIntervalDayTime interval, Timestamp ts) {
+ if (ts == null || interval == null) {
+ return null;
+ }
+
+ Timestamp tsResult = new Timestamp(0);
+ add(interval, ts, tsResult);
return tsResult;
}
+ public boolean add(HiveIntervalDayTime interval, Timestamp ts,
+ Timestamp result) {
+ if (ts == null || interval == null) {
+ return false;
+ }
+
+ nanosResult.addNanos(ts.getNanos(), interval.getNanos());
+
+ long newMillis = ts.getTime()
+ + TimeUnit.SECONDS.toMillis(interval.getTotalSeconds() + nanosResult.seconds);
+ result.setTime(newMillis);
+ result.setNanos(nanosResult.nanos);
+ return true;
+ }
+
public HiveIntervalDayTime add(HiveIntervalDayTime left, HiveIntervalDayTime right) {
- HiveIntervalDayTime result = null;
if (left == null || right == null) {
return null;
}
+ HiveIntervalDayTime result = new HiveIntervalDayTime();
+ add(left, right, result);
+
+ return result;
+ }
+
+ public boolean add(HiveIntervalDayTime left, HiveIntervalDayTime right,
+ HiveIntervalDayTime result) {
+ if (left == null || right == null) {
+ return false;
+ }
+
nanosResult.addNanos(left.getNanos(), right.getNanos());
long totalSeconds = left.getTotalSeconds() + right.getTotalSeconds() + nanosResult.seconds;
- result = new HiveIntervalDayTime(totalSeconds, nanosResult.nanos);
- return result;
+ result.set(totalSeconds, nanosResult.nanos);
+ return true;
}
public Timestamp subtract(Timestamp left, HiveIntervalDayTime right) {
@@ -212,6 +332,13 @@ public class DateTimeMath {
return add(left, right.negate());
}
+ public boolean subtract(Timestamp left, HiveIntervalDayTime right, Timestamp result) {
+ if (left == null || right == null) {
+ return false;
+ }
+ return add(left, right.negate(), result);
+ }
+
public HiveIntervalDayTime subtract(HiveIntervalDayTime left, HiveIntervalDayTime right) {
if (left == null || right == null) {
return null;
@@ -219,17 +346,36 @@ public class DateTimeMath {
return add(left, right.negate());
}
+ public boolean subtract(HiveIntervalDayTime left, HiveIntervalDayTime right,
+ HiveIntervalDayTime result) {
+ if (left == null || right == null) {
+ return false;
+ }
+ return add(left, right.negate(), result);
+ }
+
public HiveIntervalDayTime subtract(Timestamp left, Timestamp right) {
- HiveIntervalDayTime result = null;
if (left == null || right == null) {
return null;
}
+ HiveIntervalDayTime result = new HiveIntervalDayTime();
+ subtract(left, right, result);
+
+ return result;
+ }
+
+ public boolean subtract(Timestamp left, Timestamp right,
+ HiveIntervalDayTime result) {
+ if (left == null || right == null) {
+ return false;
+ }
+
nanosResult.addNanos(left.getNanos(), -(right.getNanos()));
long totalSeconds = TimeUnit.MILLISECONDS.toSeconds(left.getTime())
- TimeUnit.MILLISECONDS.toSeconds(right.getTime()) + nanosResult.seconds;
- result = new HiveIntervalDayTime(totalSeconds, nanosResult.nanos);
- return result;
+ result.set(totalSeconds, nanosResult.nanos);
+ return true;
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestTimestampWritableAndColumnVector.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestTimestampWritableAndColumnVector.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestTimestampWritableAndColumnVector.java
new file mode 100644
index 0000000..6c46257
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestTimestampWritableAndColumnVector.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector;
+
+import org.junit.Test;
+
+import java.math.BigDecimal;
+import java.math.RoundingMode;
+import java.sql.Timestamp;
+import java.util.Date;
+import java.util.Random;
+
+import org.apache.hadoop.hive.common.type.RandomTypeUtil;
+import org.apache.hadoop.hive.serde2.io.TimestampWritable;
+
+import static org.junit.Assert.*;
+
+/**
+ * Test for ListColumnVector
+ */
+public class TestTimestampWritableAndColumnVector {
+
+ private static int TEST_COUNT = 5000;
+
+ private static int fake = 0;
+
+ @Test
+ public void testDouble() throws Exception {
+
+ Random r = new Random(1234);
+ TimestampColumnVector timestampColVector = new TimestampColumnVector();
+ Timestamp[] randTimestamps = new Timestamp[VectorizedRowBatch.DEFAULT_SIZE];
+
+ for (int i = 0; i < VectorizedRowBatch.DEFAULT_SIZE; i++) {
+ Timestamp randTimestamp = RandomTypeUtil.getRandTimestamp(r);
+ randTimestamps[i] = randTimestamp;
+ timestampColVector.set(i, randTimestamp);
+ }
+ for (int i = 0; i < VectorizedRowBatch.DEFAULT_SIZE; i++) {
+ Timestamp retrievedTimestamp = timestampColVector.asScratchTimestamp(i);
+ Timestamp randTimestamp = randTimestamps[i];
+ if (!retrievedTimestamp.equals(randTimestamp)) {
+ assertTrue(false);
+ }
+ double randDouble = TimestampWritable.getDouble(randTimestamp);
+ double retrievedDouble = timestampColVector.getDouble(i);
+ if (randDouble != retrievedDouble) {
+ assertTrue(false);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorExpressionWriters.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorExpressionWriters.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorExpressionWriters.java
index fc38dd3..02602f4 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorExpressionWriters.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorExpressionWriters.java
@@ -229,18 +229,19 @@ public class TestVectorExpressionWriters {
}
private void testWriterTimestamp(TypeInfo type) throws HiveException {
- TimestampColumnVector tcv = VectorizedRowGroupGenUtil.generateTimestampColumnVector(true, false,
- vectorSize, new Random(10));
+ Timestamp[] timestampValues = new Timestamp[vectorSize];
+ TimestampColumnVector tcv =
+ VectorizedRowGroupGenUtil.generateTimestampColumnVector(true, false,
+ vectorSize, new Random(10), timestampValues);
tcv.isNull[3] = true;
VectorExpressionWriter vew = getWriter(type);
for (int i = 0; i < vectorSize; i++) {
Writable w = (Writable) vew.writeValue(tcv, i);
if (w != null) {
- Writable expected = getWritableValue(type, tcv.asScratchTimestamp(i));
+ Writable expected = getWritableValue(type, timestampValues[i]);
TimestampWritable t1 = (TimestampWritable) expected;
TimestampWritable t2 = (TimestampWritable) w;
- Assert.assertTrue(t1.getNanos() == t2.getNanos());
- Assert.assertTrue(t1.getSeconds() == t2.getSeconds());
+ Assert.assertTrue(t1.equals(t2));
} else {
Assert.assertTrue(tcv.isNull[i]);
}
@@ -248,8 +249,10 @@ public class TestVectorExpressionWriters {
}
private void testSetterTimestamp(TypeInfo type) throws HiveException {
- TimestampColumnVector tcv = VectorizedRowGroupGenUtil.generateTimestampColumnVector(true, false,
- vectorSize, new Random(10));
+ Timestamp[] timestampValues = new Timestamp[vectorSize];
+ TimestampColumnVector tcv =
+ VectorizedRowGroupGenUtil.generateTimestampColumnVector(true, false,
+ vectorSize, new Random(10), timestampValues);
tcv.isNull[3] = true;
Object[] values = new Object[this.vectorSize];
@@ -259,12 +262,10 @@ public class TestVectorExpressionWriters {
values[i] = null; // setValue() should be able to handle null input
values[i] = vew.setValue(values[i], tcv, i);
if (values[i] != null) {
- Timestamp scratchTimestamp = tcv.asScratchTimestamp(i);
- Writable expected = getWritableValue(type, scratchTimestamp);
+ Writable expected = getWritableValue(type, timestampValues[i]);
TimestampWritable t1 = (TimestampWritable) expected;
TimestampWritable t2 = (TimestampWritable) values[i];
- Assert.assertTrue(t1.getNanos() == t2.getNanos());
- Assert.assertTrue(t1.getSeconds() == t2.getSeconds());
+ Assert.assertTrue(t1.equals(t2));
} else {
Assert.assertTrue(tcv.isNull[i]);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorFilterExpressions.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorFilterExpressions.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorFilterExpressions.java
index 819cc27..80f55dc 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorFilterExpressions.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorFilterExpressions.java
@@ -25,7 +25,6 @@ import static org.junit.Assert.assertTrue;
import java.sql.Timestamp;
import org.apache.hadoop.hive.common.type.HiveDecimal;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorMathFunctions.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorMathFunctions.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorMathFunctions.java
index c14eb4a..31add6e 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorMathFunctions.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorMathFunctions.java
@@ -19,11 +19,13 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions;
import java.io.UnsupportedEncodingException;
+import java.sql.Timestamp;
import java.util.Arrays;
+import java.util.Random;
import junit.framework.Assert;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
+import org.apache.hadoop.hive.common.type.RandomTypeUtil;
import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
@@ -52,6 +54,7 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FuncSignLongToDoubl
import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FuncSinDoubleToDouble;
import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FuncSqrtDoubleToDouble;
import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FuncTanDoubleToDouble;
+import org.apache.hadoop.hive.serde2.io.TimestampWritable;
import org.junit.Test;
@@ -194,22 +197,22 @@ public class TestVectorMathFunctions {
return batch;
}
- public static VectorizedRowBatch getVectorizedRowBatchTimestampInDoubleOut() {
+ public static VectorizedRowBatch getVectorizedRowBatchTimestampInDoubleOut(double[] doubleValues) {
+ Random r = new Random(45993);
VectorizedRowBatch batch = new VectorizedRowBatch(2);
TimestampColumnVector tcv;
DoubleColumnVector dcv;
- tcv = new TimestampColumnVector();
- dcv = new DoubleColumnVector();
- tcv.set(0, new PisaTimestamp(0, -2));
- tcv.set(1, new PisaTimestamp(0, -1));
- tcv.set(2, new PisaTimestamp(0, 0));
- tcv.set(3, new PisaTimestamp(0, 1));
- tcv.set(4, new PisaTimestamp(0, 2));
+ tcv = new TimestampColumnVector(doubleValues.length);
+ dcv = new DoubleColumnVector(doubleValues.length);
+ for (int i = 0; i < doubleValues.length; i++) {
+ doubleValues[i] = r.nextDouble() % (double) SECONDS_LIMIT;
+ dcv.vector[i] = doubleValues[i];
+ }
batch.cols[0] = tcv;
batch.cols[1] = dcv;
- batch.size = 5;
+ batch.size = doubleValues.length;
return batch;
}
@@ -228,35 +231,45 @@ public class TestVectorMathFunctions {
return batch;
}
- public static VectorizedRowBatch getVectorizedRowBatchTimestampInLongOut() {
+ public static VectorizedRowBatch getVectorizedRowBatchTimestampInLongOut(long[] longValues) {
+ Random r = new Random(345);
VectorizedRowBatch batch = new VectorizedRowBatch(2);
TimestampColumnVector inV;
LongColumnVector outV;
- inV = new TimestampColumnVector();
- outV = new LongColumnVector();
- inV.setTimestampSeconds(0, 2);
- inV.setTimestampSeconds(1, 2);
+ inV = new TimestampColumnVector(longValues.length);
+ outV = new LongColumnVector(longValues.length);
+ for (int i = 0; i < longValues.length; i++) {
+ Timestamp randTimestamp = RandomTypeUtil.getRandTimestamp(r);
+ longValues[i] = TimestampWritable.getLong(randTimestamp);
+ inV.set(0, randTimestamp);
+ }
batch.cols[0] = inV;
batch.cols[1] = outV;
- batch.size = 2;
+ batch.size = longValues.length;
return batch;
}
- public static VectorizedRowBatch getVectorizedRowBatchLongInTimestampOut() {
+ static long SECONDS_LIMIT = 60L * 24L * 365L * 9999L;
+
+ public static VectorizedRowBatch getVectorizedRowBatchLongInTimestampOut(long[] longValues) {
+ Random r = new Random(12099);
VectorizedRowBatch batch = new VectorizedRowBatch(2);
LongColumnVector inV;
TimestampColumnVector outV;
inV = new LongColumnVector();
outV = new TimestampColumnVector();
- inV.vector[0] = -2;
- inV.vector[1] = 2;
+
+ for (int i = 0; i < longValues.length; i++) {
+ longValues[i] = r.nextLong() % SECONDS_LIMIT;
+ inV.vector[i] = longValues[i];
+ }
batch.cols[0] = inV;
batch.cols[1] = outV;
- batch.size = 2;
+ batch.size = longValues.length;
return batch;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTimestampExpressions.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTimestampExpressions.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTimestampExpressions.java
index 375f369..d4f1f6f 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTimestampExpressions.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTimestampExpressions.java
@@ -32,7 +32,6 @@ import java.util.Random;
import junit.framework.Assert;
import org.apache.commons.lang.ArrayUtils;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
import org.apache.hadoop.hive.common.type.RandomTypeUtil;
import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
@@ -84,7 +83,7 @@ public class TestVectorTimestampExpressions {
}
private Timestamp[] getAllBoundaries() {
- return getAllBoundaries(0000, 9999);
+ return getAllBoundaries(RandomTypeUtil.MIN_YEAR, RandomTypeUtil.MAX_YEAR);
}
private VectorizedRowBatch getVectorizedRandomRowBatchTimestampLong(int seed, int size) {
@@ -742,27 +741,14 @@ public class TestVectorTimestampExpressions {
testVectorUDFSecond(TestType.STRING_LONG);
}
- private LongWritable getLongWritable(TimestampWritable i) {
- LongWritable result = new LongWritable();
- if (i == null) {
- return null;
- } else {
- result.set(i.getSeconds());
- return result;
+ private void compareToUDFUnixTimeStampLong(Timestamp ts, long y) {
+ long seconds = ts.getTime() / 1000;
+ if(seconds != y) {
+ System.out.printf("%d vs %d for %s\n", seconds, y, ts.toString());
+ Assert.assertTrue(false);
}
}
- private void compareToUDFUnixTimeStampLong(Timestamp t, long y) {
- TimestampWritable tsw = new TimestampWritable(t);
- LongWritable res = getLongWritable(tsw);
- if(res.get() != y) {
- System.out.printf("%d vs %d for %s, %d\n", res.get(), y, t.toString(),
- tsw.getTimestamp().getTime()/1000);
- }
-
- Assert.assertEquals(res.get(), y);
- }
-
private void verifyUDFUnixTimeStamp(VectorizedRowBatch batch, TestType testType) {
VectorExpression udf;
if (testType == TestType.TIMESTAMP_LONG) {
[05/50] [abbrv] hive git commit: HIVE-13151 : Clean up UGI objects in
FileSystem cache for transactions (Wei Zheng, reviewed by Eugene Koifman)
Posted by jd...@apache.org.
HIVE-13151 : Clean up UGI objects in FileSystem cache for transactions (Wei Zheng, reviewed by Eugene Koifman)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f9d1b6ab
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f9d1b6ab
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f9d1b6ab
Branch: refs/heads/llap
Commit: f9d1b6ab77ab15b8337c17fbe38557c1f7b5ce58
Parents: d3a5f20
Author: Wei Zheng <we...@apache.org>
Authored: Thu Mar 24 17:29:59 2016 -0700
Committer: Wei Zheng <we...@apache.org>
Committed: Thu Mar 24 17:29:59 2016 -0700
----------------------------------------------------------------------
.../hive/hcatalog/streaming/HiveEndPoint.java | 11 +++++
.../hadoop/hive/ql/txn/compactor/Cleaner.java | 5 +++
.../hive/ql/txn/compactor/CompactorThread.java | 5 +++
.../hadoop/hive/ql/txn/compactor/Initiator.java | 9 +++-
.../hadoop/hive/ql/txn/compactor/Worker.java | 8 +++-
.../apache/hadoop/hive/ql/TestTxnCommands2.java | 47 ++++++++++++++++++++
6 files changed, 82 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/f9d1b6ab/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
----------------------------------------------------------------------
diff --git a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
index 4c77842..baeafad 100644
--- a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
+++ b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
@@ -18,6 +18,7 @@
package org.apache.hive.hcatalog.streaming;
+import org.apache.hadoop.fs.FileSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.hive.cli.CliSessionState;
@@ -342,6 +343,11 @@ public class HiveEndPoint {
return null;
}
} );
+ try {
+ FileSystem.closeAllForUGI(ugi);
+ } catch (IOException exception) {
+ LOG.error("Could not clean up file-system handles for UGI: " + ugi, exception);
+ }
} catch (IOException e) {
LOG.error("Error closing connection to " + endPt, e);
} catch (InterruptedException e) {
@@ -937,6 +943,11 @@ public class HiveEndPoint {
}
}
);
+ try {
+ FileSystem.closeAllForUGI(ugi);
+ } catch (IOException exception) {
+ LOG.error("Could not clean up file-system handles for UGI: " + ugi, exception);
+ }
} catch (IOException e) {
throw new ImpersonationFailed("Failed closing Txn Batch as user '" + username +
"' on endPoint :" + endPt, e);
http://git-wip-us.apache.org/repos/asf/hive/blob/f9d1b6ab/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java
index 9ffeaec..4c31a49 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java
@@ -272,6 +272,11 @@ public class Cleaner extends CompactorThread {
return null;
}
});
+ try {
+ FileSystem.closeAllForUGI(ugi);
+ } catch (IOException exception) {
+ LOG.error("Could not clean up file-system handles for UGI: " + ugi, exception + " for " +
+ ci.getFullPartitionName()); }
}
txnHandler.markCleaned(ci);
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/hive/blob/f9d1b6ab/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java
index 8495c66..4d6e24e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java
@@ -174,6 +174,11 @@ abstract class CompactorThread extends Thread implements MetaStoreThread {
return null;
}
});
+ try {
+ FileSystem.closeAllForUGI(ugi);
+ } catch (IOException exception) {
+ LOG.error("Could not clean up file-system handles for UGI: " + ugi, exception);
+ }
if (wrapper.size() == 1) {
LOG.debug("Running job as " + wrapper.get(0));
http://git-wip-us.apache.org/repos/asf/hive/blob/f9d1b6ab/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
index 916d9dc..98ebf53 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
@@ -226,12 +226,19 @@ public class Initiator extends CompactorThread {
LOG.info("Going to initiate as user " + runAs);
UserGroupInformation ugi = UserGroupInformation.createProxyUser(runAs,
UserGroupInformation.getLoginUser());
- return ugi.doAs(new PrivilegedExceptionAction<CompactionType>() {
+ CompactionType compactionType = ugi.doAs(new PrivilegedExceptionAction<CompactionType>() {
@Override
public CompactionType run() throws Exception {
return determineCompactionType(ci, txns, sd);
}
});
+ try {
+ FileSystem.closeAllForUGI(ugi);
+ } catch (IOException exception) {
+ LOG.error("Could not clean up file-system handles for UGI: " + ugi, exception + " for " +
+ ci.getFullPartitionName());
+ }
+ return compactionType;
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/f9d1b6ab/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
index adffa8c..e21ca27 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hive.ql.txn.compactor;
+import org.apache.hadoop.fs.FileSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.hive.common.ValidTxnList;
@@ -34,8 +35,6 @@ import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.net.InetAddress;
@@ -173,6 +172,11 @@ public class Worker extends CompactorThread {
return null;
}
});
+ try {
+ FileSystem.closeAllForUGI(ugi);
+ } catch (IOException exception) {
+ LOG.error("Could not clean up file-system handles for UGI: " + ugi, exception + " for " +
+ ci.getFullPartitionName()); }
}
txnHandler.markCompacted(ci);
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/hive/blob/f9d1b6ab/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
index 0786c21..04c1d17 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
@@ -51,10 +51,13 @@ import org.junit.Test;
import org.junit.rules.TestName;
import java.io.File;
+import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -546,6 +549,50 @@ public class TestTxnCommands2 {
Assert.assertEquals("Unexpected num succeeded", 1, cbs.succeeded);
Assert.assertEquals("Unexpected num total5", hiveConf.getIntVar(HiveConf.ConfVars.COMPACTOR_HISTORY_RETENTION_FAILED) + 1, cbs.total);
}
+
+ /**
+ * Make sure there's no FileSystem$Cache$Key leak due to UGI use
+ * @throws Exception
+ */
+ @Test
+ public void testFileSystemUnCaching() throws Exception {
+ int cacheSizeBefore;
+ int cacheSizeAfter;
+
+ // get the size of cache BEFORE
+ cacheSizeBefore = getFileSystemCacheSize();
+
+ // Insert a row to ACID table
+ runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) values(1,2)");
+
+ // Perform a major compaction
+ runStatementOnDriver("alter table " + Table.ACIDTBL + " compact 'major'");
+ runWorker(hiveConf);
+ runCleaner(hiveConf);
+
+ // get the size of cache AFTER
+ cacheSizeAfter = getFileSystemCacheSize();
+
+ Assert.assertEquals(cacheSizeBefore, cacheSizeAfter);
+ }
+
+ private int getFileSystemCacheSize() throws Exception {
+ try {
+ Field cache = FileSystem.class.getDeclaredField("CACHE");
+ cache.setAccessible(true);
+ Object o = cache.get(null); // FileSystem.CACHE
+
+ Field mapField = o.getClass().getDeclaredField("map");
+ mapField.setAccessible(true);
+ Map map = (HashMap)mapField.get(o); // FileSystem.CACHE.map
+
+ return map.size();
+ } catch (NoSuchFieldException e) {
+ System.out.println(e);
+ }
+ return 0;
+ }
+
private static class CompactionsByState {
private int attempted;
private int failed;
[35/50] [abbrv] hive git commit: HIVE-13111: Fix timestamp /
interval_day_time wrong results with HIVE-9862 (Matt McCline,
reviewed by Jason Dere)
Posted by jd...@apache.org.
HIVE-13111: Fix timestamp / interval_day_time wrong results with HIVE-9862 (Matt McCline, reviewed by Jason Dere)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/52016296
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/52016296
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/52016296
Branch: refs/heads/llap
Commit: 52016296cf7de89d868e1073a3376b330fee0955
Parents: 761b547
Author: Matt McCline <mm...@hortonworks.com>
Authored: Mon Mar 28 14:14:37 2016 -0700
Committer: Matt McCline <mm...@hortonworks.com>
Committed: Mon Mar 28 14:14:37 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/ant/GenVectorCode.java | 531 ++++-----
.../hive/common/type/HiveIntervalDayTime.java | 245 ----
.../org/apache/hive/common/util/DateUtils.java | 19 -
data/files/timestamps.txt | 50 +
.../test/resources/testconfiguration.properties | 2 +
.../java/org/apache/orc/impl/WriterImpl.java | 23 +-
...eColumnArithmeticIntervalYearMonthColumn.txt | 56 +-
...eColumnArithmeticIntervalYearMonthScalar.txt | 55 +-
.../DateColumnArithmeticTimestampColumn.txt | 141 ++-
.../DateColumnArithmeticTimestampColumnBase.txt | 171 ---
.../DateColumnArithmeticTimestampScalar.txt | 113 +-
.../DateColumnArithmeticTimestampScalarBase.txt | 137 ---
...eScalarArithmeticIntervalYearMonthColumn.txt | 53 +-
.../DateScalarArithmeticTimestampColumn.txt | 108 +-
.../DateScalarArithmeticTimestampColumnBase.txt | 147 ---
...ayTimeColumnCompareIntervalDayTimeColumn.txt | 52 -
...ayTimeColumnCompareIntervalDayTimeScalar.txt | 55 -
...ayTimeScalarCompareIntervalDayTimeColumn.txt | 55 -
...erLongDoubleColumnCompareTimestampColumn.txt | 2 +-
...erLongDoubleColumnCompareTimestampScalar.txt | 4 +-
...erLongDoubleScalarCompareTimestampColumn.txt | 4 +
.../FilterTimestampColumnBetween.txt | 11 +-
...terTimestampColumnCompareTimestampColumn.txt | 417 ++++++-
...imestampColumnCompareTimestampColumnBase.txt | 429 -------
...terTimestampColumnCompareTimestampScalar.txt | 128 ++-
...imestampColumnCompareTimestampScalarBase.txt | 145 ---
...erTimestampScalarCompareLongDoubleColumn.txt | 3 +-
...terTimestampScalarCompareTimestampColumn.txt | 132 ++-
...imestampScalarCompareTimestampColumnBase.txt | 147 ---
...ayTimeColumnCompareIntervalDayTimeColumn.txt | 54 -
...ayTimeColumnCompareIntervalDayTimeScalar.txt | 57 -
...ayTimeScalarCompareIntervalDayTimeColumn.txt | 57 -
...ervalYearMonthColumnArithmeticDateColumn.txt | 55 +-
...ervalYearMonthColumnArithmeticDateScalar.txt | 51 +-
...YearMonthColumnArithmeticTimestampColumn.txt | 63 +-
...YearMonthColumnArithmeticTimestampScalar.txt | 48 +-
...ervalYearMonthScalarArithmeticDateColumn.txt | 51 +-
...YearMonthScalarArithmeticTimestampColumn.txt | 55 +-
.../LongDoubleColumnCompareTimestampColumn.txt | 1 -
.../LongDoubleColumnCompareTimestampScalar.txt | 3 +-
.../LongDoubleScalarCompareTimestampColumn.txt | 1 +
.../TimestampColumnArithmeticDateColumn.txt | 138 ++-
.../TimestampColumnArithmeticDateColumnBase.txt | 172 ---
.../TimestampColumnArithmeticDateScalar.txt | 98 +-
.../TimestampColumnArithmeticDateScalarBase.txt | 126 --
...pColumnArithmeticIntervalYearMonthColumn.txt | 59 +-
...pColumnArithmeticIntervalYearMonthScalar.txt | 41 +-
...TimestampColumnArithmeticTimestampColumn.txt | 128 ++-
...stampColumnArithmeticTimestampColumnBase.txt | 152 ---
...TimestampColumnArithmeticTimestampScalar.txt | 96 +-
...stampColumnArithmeticTimestampScalarBase.txt | 125 --
.../TimestampColumnCompareLongDoubleScalar.txt | 1 +
.../TimestampColumnCompareTimestampColumn.txt | 122 +-
...imestampColumnCompareTimestampColumnBase.txt | 140 ---
.../TimestampColumnCompareTimestampScalar.txt | 114 +-
...imestampColumnCompareTimestampScalarBase.txt | 131 ---
.../TimestampScalarArithmeticDateColumn.txt | 117 +-
.../TimestampScalarArithmeticDateColumnBase.txt | 151 ---
...pScalarArithmeticIntervalYearMonthColumn.txt | 62 +-
...TimestampScalarArithmeticTimestampColumn.txt | 103 +-
...stampScalarArithmeticTimestampColumnBase.txt | 139 ---
.../TimestampScalarCompareLongDoubleColumn.txt | 4 +-
.../TimestampScalarCompareTimestampColumn.txt | 115 +-
...imestampScalarCompareTimestampColumnBase.txt | 132 ---
.../VectorUDAFMinMaxIntervalDayTime.txt | 454 ++++++++
.../UDAFTemplates/VectorUDAFMinMaxTimestamp.txt | 31 +-
.../hive/ql/exec/vector/TimestampUtils.java | 8 +
.../hive/ql/exec/vector/VectorAssignRow.java | 23 +-
.../exec/vector/VectorColumnAssignFactory.java | 19 +-
.../ql/exec/vector/VectorColumnSetInfo.java | 23 +-
.../hive/ql/exec/vector/VectorCopyRow.java | 32 +
.../ql/exec/vector/VectorDeserializeRow.java | 13 +-
.../exec/vector/VectorExpressionDescriptor.java | 6 +-
.../hive/ql/exec/vector/VectorExtractRow.java | 24 +-
.../ql/exec/vector/VectorGroupKeyHelper.java | 12 +
.../ql/exec/vector/VectorHashKeyWrapper.java | 82 +-
.../exec/vector/VectorHashKeyWrapperBatch.java | 112 +-
.../hive/ql/exec/vector/VectorSerializeRow.java | 7 +-
.../ql/exec/vector/VectorizationContext.java | 13 +-
.../ql/exec/vector/VectorizedBatchUtil.java | 20 +-
.../ql/exec/vector/VectorizedRowBatchCtx.java | 12 +-
.../expressions/CastDecimalToTimestamp.java | 3 +-
.../expressions/CastDoubleToTimestamp.java | 17 +-
.../vector/expressions/CastLongToTimestamp.java | 8 +-
.../CastMillisecondsLongToTimestamp.java | 22 +-
.../CastStringToIntervalDayTime.java | 8 +-
.../expressions/CastTimestampToBoolean.java | 4 +-
.../vector/expressions/CastTimestampToDate.java | 2 +-
.../expressions/CastTimestampToDecimal.java | 9 +-
.../expressions/CastTimestampToDouble.java | 13 +-
.../vector/expressions/CastTimestampToLong.java | 12 +-
.../expressions/ConstantVectorExpression.java | 36 +-
.../expressions/DateColSubtractDateColumn.java | 80 +-
.../expressions/DateColSubtractDateScalar.java | 51 +-
.../DateScalarSubtractDateColumn.java | 52 +-
.../FilterTimestampColumnInList.java | 27 +-
.../IfExprIntervalDayTimeColumnColumn.java | 103 +-
.../IfExprIntervalDayTimeColumnScalar.java | 94 +-
.../IfExprIntervalDayTimeScalarColumn.java | 96 +-
.../IfExprIntervalDayTimeScalarScalar.java | 84 +-
.../IfExprTimestampColumnColumnBase.java | 8 +-
.../IfExprTimestampColumnScalar.java | 3 +-
.../IfExprTimestampColumnScalarBase.java | 14 +-
.../IfExprTimestampScalarColumn.java | 3 +-
.../IfExprTimestampScalarColumnBase.java | 15 +-
.../IfExprTimestampScalarScalar.java | 3 +-
.../IfExprTimestampScalarScalarBase.java | 13 +-
.../ql/exec/vector/expressions/NullUtil.java | 26 +
.../expressions/TimestampColumnInList.java | 29 +-
.../expressions/VectorExpressionWriter.java | 6 +-
.../VectorExpressionWriterFactory.java | 124 +-
.../expressions/VectorUDFDateAddColCol.java | 2 +-
.../expressions/VectorUDFDateAddColScalar.java | 2 +-
.../expressions/VectorUDFDateDiffColCol.java | 10 +-
.../expressions/VectorUDFDateDiffColScalar.java | 2 +-
.../expressions/VectorUDFDateDiffScalarCol.java | 2 +-
.../expressions/VectorUDFDateTimestamp.java | 2 +-
.../expressions/VectorUDFUnixTimeStampDate.java | 7 +-
.../VectorUDFUnixTimeStampTimestamp.java | 5 +-
.../aggregates/VectorUDAFAvgTimestamp.java | 40 +-
.../aggregates/VectorUDAFStdPopTimestamp.java | 24 +-
.../aggregates/VectorUDAFStdSampTimestamp.java | 27 +-
.../aggregates/VectorUDAFVarPopTimestamp.java | 24 +-
.../aggregates/VectorUDAFVarSampTimestamp.java | 24 +-
.../ql/exec/vector/udf/VectorUDFAdaptor.java | 13 +-
.../hadoop/hive/ql/util/DateTimeMath.java | 214 +++-
.../TestTimestampWritableAndColumnVector.java | 68 ++
.../TestVectorExpressionWriters.java | 23 +-
.../TestVectorFilterExpressions.java | 1 -
.../expressions/TestVectorMathFunctions.java | 53 +-
.../TestVectorTimestampExpressions.java | 26 +-
.../vector/expressions/TestVectorTypeCasts.java | 216 ++--
.../FakeVectorRowBatchFromObjectIterables.java | 3 +-
.../vector/util/VectorizedRowGroupGenUtil.java | 14 +-
.../hive/ql/io/orc/TestInputOutputFormat.java | 2 +-
.../hadoop/hive/ql/io/orc/TestOrcFile.java | 5 +-
.../hive/ql/io/orc/TestVectorOrcFile.java | 22 +-
.../clientpositive/vector_interval_arithmetic.q | 174 +++
.../tez/vector_interval_arithmetic.q.out | 1086 ++++++++++++++++++
.../clientpositive/tez/vectorized_casts.q.out | 18 +-
.../tez/vectorized_timestamp.q.out | 157 +++
.../vector_interval_arithmetic.q.out | 1027 +++++++++++++++++
.../clientpositive/vectorized_casts.q.out | 18 +-
.../hive/serde2/io/TimestampWritable.java | 71 +-
.../hive/common/type/HiveIntervalDayTime.java | 253 ++++
.../hadoop/hive/common/type/PisaTimestamp.java | 609 ----------
.../hadoop/hive/common/type/RandomTypeUtil.java | 70 +-
.../hive/ql/exec/vector/ColumnVector.java | 2 +-
.../vector/IntervalDayTimeColumnVector.java | 348 ++++++
.../ql/exec/vector/TimestampColumnVector.java | 341 ++----
.../hive/common/util/IntervalDayTimeUtils.java | 77 ++
.../hive/common/type/TestPisaTimestamp.java | 118 --
.../exec/vector/TestTimestampColumnVector.java | 117 ++
153 files changed, 8221 insertions(+), 5349 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java
----------------------------------------------------------------------
diff --git a/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java b/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java
index 2e369ec..6c6cc63 100644
--- a/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java
+++ b/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java
@@ -77,15 +77,6 @@ public class GenVectorCode extends Task {
{"DTIScalarArithmeticDTIColumnNoConvert", "Subtract", "interval_year_month", "interval_year_month", "-"},
{"DTIColumnArithmeticDTIColumnNoConvert", "Subtract", "interval_year_month", "interval_year_month", "-"},
- // Arithmetic on two TimestampColumnVector base classes.
- {"TimestampArithmeticTimestampBase", "Add", "Col", "Column"},
- {"TimestampArithmeticTimestampBase", "Add", "Scalar", "Column"},
- {"TimestampArithmeticTimestampBase", "Add", "Col", "Scalar"},
-
- {"TimestampArithmeticTimestampBase", "Subtract", "Col", "Column"},
- {"TimestampArithmeticTimestampBase", "Subtract", "Scalar", "Column"},
- {"TimestampArithmeticTimestampBase", "Subtract", "Col", "Scalar"},
-
// Arithmetic on two type interval_day_time (TimestampColumnVector storing nanosecond interval
// in 2 longs) produces a interval_day_time.
{"TimestampArithmeticTimestamp", "Add", "interval_day_time", "Col", "interval_day_time", "Scalar"},
@@ -111,30 +102,13 @@ public class GenVectorCode extends Task {
{"TimestampArithmeticTimestamp", "Subtract", "timestamp", "Col", "interval_day_time", "Column"},
// A type timestamp (TimestampColumnVector) minus a type timestamp produces a
- // type interval_day_time (TimestampColumnVector storing nanosecond interval in 2 longs).
+ // type interval_day_time (IntervalDayTimeColumnVector storing nanosecond interval in 2 primitives).
{"TimestampArithmeticTimestamp", "Subtract", "timestamp", "Col", "timestamp", "Scalar"},
{"TimestampArithmeticTimestamp", "Subtract", "timestamp", "Scalar", "timestamp", "Column"},
{"TimestampArithmeticTimestamp", "Subtract", "timestamp", "Col", "timestamp", "Column"},
- // Arithmetic on a TimestampColumnVector and date base classes.
- {"DateArithmeticTimestampBase", "Add", "Col", "Column"},
- {"DateArithmeticTimestampBase", "Add", "Scalar", "Column"},
- {"DateArithmeticTimestampBase", "Add", "Col", "Scalar"},
-
- {"DateArithmeticTimestampBase", "Subtract", "Col", "Column"},
- {"DateArithmeticTimestampBase", "Subtract", "Scalar", "Column"},
- {"DateArithmeticTimestampBase", "Subtract", "Col", "Scalar"},
-
- {"TimestampArithmeticDateBase", "Add", "Col", "Column"},
- {"TimestampArithmeticDateBase", "Add", "Scalar", "Column"},
- {"TimestampArithmeticDateBase", "Add", "Col", "Scalar"},
-
- {"TimestampArithmeticDateBase", "Subtract", "Col", "Column"},
- {"TimestampArithmeticDateBase", "Subtract", "Scalar", "Column"},
- {"TimestampArithmeticDateBase", "Subtract", "Col", "Scalar"},
-
- // Arithmetic with a type date (LongColumnVector storing epoch days) and type interval_day_time (TimestampColumnVector storing
- // nanosecond interval in 2 longs) produces a type timestamp (TimestampColumnVector).
+ // Arithmetic with a type date (LongColumnVector storing epoch days) and type interval_day_time (IntervalDayTimeColumnVector storing
+ // nanosecond interval in 2 primitives) produces a type timestamp (TimestampColumnVector).
{"DateArithmeticTimestamp", "Add", "date", "Col", "interval_day_time", "Column"},
{"DateArithmeticTimestamp", "Add", "date", "Scalar", "interval_day_time", "Column"},
{"DateArithmeticTimestamp", "Add", "date", "Col", "interval_day_time", "Scalar"},
@@ -147,7 +121,8 @@ public class GenVectorCode extends Task {
{"TimestampArithmeticDate", "Add", "interval_day_time", "Scalar", "date", "Column"},
{"TimestampArithmeticDate", "Add", "interval_day_time", "Col", "date", "Scalar"},
- // Subtraction with a type date (LongColumnVector storing epoch days) and type timestamp produces a type timestamp (TimestampColumnVector).
+ // Subtraction with a type date (LongColumnVector storing days) and type timestamp produces a
+ // type interval_day_time (IntervalDayTimeColumnVector).
{"DateArithmeticTimestamp", "Subtract", "date", "Col", "timestamp", "Column"},
{"DateArithmeticTimestamp", "Subtract", "date", "Scalar", "timestamp", "Column"},
{"DateArithmeticTimestamp", "Subtract", "date", "Col", "timestamp", "Scalar"},
@@ -318,70 +293,48 @@ public class GenVectorCode extends Task {
{"ScalarCompareColumn", "Greater", "double", "long", ">"},
{"ScalarCompareColumn", "GreaterEqual", "double", "long", ">="},
- // Base compare timestamp to timestamp used by Timestamp and IntervalDayTime.
- {"TimestampCompareTimestampBase", "Equal", "==", "Col", "Column"},
- {"TimestampCompareTimestampBase", "NotEqual", "!=", "Col", "Column"},
- {"TimestampCompareTimestampBase", "Less", "<", "Col", "Column"},
- {"TimestampCompareTimestampBase", "LessEqual", "<=", "Col", "Column"},
- {"TimestampCompareTimestampBase", "Greater", ">", "Col", "Column"},
- {"TimestampCompareTimestampBase", "GreaterEqual", ">=", "Col", "Column"},
-
- {"TimestampCompareTimestampBase", "Equal", "==", "Col", "Scalar"},
- {"TimestampCompareTimestampBase", "NotEqual", "!=", "Col", "Scalar"},
- {"TimestampCompareTimestampBase", "Less", "<", "Col", "Scalar"},
- {"TimestampCompareTimestampBase", "LessEqual", "<=", "Col", "Scalar"},
- {"TimestampCompareTimestampBase", "Greater", ">", "Col", "Scalar"},
- {"TimestampCompareTimestampBase", "GreaterEqual", ">=", "Col", "Scalar"},
-
- {"TimestampCompareTimestampBase", "Equal", "==", "Scalar", "Column"},
- {"TimestampCompareTimestampBase", "NotEqual", "!=", "Scalar", "Column"},
- {"TimestampCompareTimestampBase", "Less", "<", "Scalar", "Column"},
- {"TimestampCompareTimestampBase", "LessEqual", "<=", "Scalar", "Column"},
- {"TimestampCompareTimestampBase", "Greater", ">", "Scalar", "Column"},
- {"TimestampCompareTimestampBase", "GreaterEqual", ">=", "Scalar", "Column"},
-
// Compare timestamp to timestamp.
- {"TimestampCompareTimestamp", "Equal", "timestamp", "Col", "Column"},
- {"TimestampCompareTimestamp", "NotEqual", "timestamp", "Col", "Column"},
- {"TimestampCompareTimestamp", "Less", "timestamp", "Col", "Column"},
- {"TimestampCompareTimestamp", "LessEqual", "timestamp", "Col", "Column"},
- {"TimestampCompareTimestamp", "Greater", "timestamp", "Col", "Column"},
- {"TimestampCompareTimestamp", "GreaterEqual", "timestamp", "Col", "Column"},
-
- {"TimestampCompareTimestamp", "Equal", "timestamp", "Col", "Scalar"},
- {"TimestampCompareTimestamp", "NotEqual", "timestamp", "Col", "Scalar"},
- {"TimestampCompareTimestamp", "Less", "timestamp", "Col", "Scalar"},
- {"TimestampCompareTimestamp", "LessEqual", "timestamp", "Col", "Scalar"},
- {"TimestampCompareTimestamp", "Greater", "timestamp", "Col", "Scalar"},
- {"TimestampCompareTimestamp", "GreaterEqual", "timestamp", "Col", "Scalar"},
-
- {"TimestampCompareTimestamp", "Equal", "timestamp", "Scalar", "Column"},
- {"TimestampCompareTimestamp", "NotEqual", "timestamp", "Scalar", "Column"},
- {"TimestampCompareTimestamp", "Less", "timestamp", "Scalar", "Column"},
- {"TimestampCompareTimestamp", "LessEqual", "timestamp", "Scalar", "Column"},
- {"TimestampCompareTimestamp", "Greater", "timestamp", "Scalar", "Column"},
- {"TimestampCompareTimestamp", "GreaterEqual", "timestamp", "Scalar", "Column"},
-
- {"TimestampCompareTimestamp", "Equal", "interval_day_time", "Col", "Column"},
- {"TimestampCompareTimestamp", "NotEqual", "interval_day_time", "Col", "Column"},
- {"TimestampCompareTimestamp", "Less", "interval_day_time", "Col", "Column"},
- {"TimestampCompareTimestamp", "LessEqual", "interval_day_time", "Col", "Column"},
- {"TimestampCompareTimestamp", "Greater", "interval_day_time", "Col", "Column"},
- {"TimestampCompareTimestamp", "GreaterEqual", "interval_day_time", "Col", "Column"},
-
- {"TimestampCompareTimestamp", "Equal", "interval_day_time", "Col", "Scalar"},
- {"TimestampCompareTimestamp", "NotEqual", "interval_day_time", "Col", "Scalar"},
- {"TimestampCompareTimestamp", "Less", "interval_day_time", "Col", "Scalar"},
- {"TimestampCompareTimestamp", "LessEqual", "interval_day_time", "Col", "Scalar"},
- {"TimestampCompareTimestamp", "Greater", "interval_day_time", "Col", "Scalar"},
- {"TimestampCompareTimestamp", "GreaterEqual", "interval_day_time", "Col", "Scalar"},
-
- {"TimestampCompareTimestamp", "Equal", "interval_day_time", "Scalar", "Column"},
- {"TimestampCompareTimestamp", "NotEqual", "interval_day_time", "Scalar", "Column"},
- {"TimestampCompareTimestamp", "Less", "interval_day_time", "Scalar", "Column"},
- {"TimestampCompareTimestamp", "LessEqual", "interval_day_time", "Scalar", "Column"},
- {"TimestampCompareTimestamp", "Greater", "interval_day_time", "Scalar", "Column"},
- {"TimestampCompareTimestamp", "GreaterEqual", "interval_day_time", "Scalar", "Column"},
+ {"TimestampCompareTimestamp", "Equal", "==", "timestamp", "Col", "Column"},
+ {"TimestampCompareTimestamp", "NotEqual", "!=", "timestamp", "Col", "Column"},
+ {"TimestampCompareTimestamp", "Less", "<", "timestamp", "Col", "Column"},
+ {"TimestampCompareTimestamp", "LessEqual", "<=", "timestamp", "Col", "Column"},
+ {"TimestampCompareTimestamp", "Greater", ">", "timestamp", "Col", "Column"},
+ {"TimestampCompareTimestamp", "GreaterEqual", ">=", "timestamp", "Col", "Column"},
+
+ {"TimestampCompareTimestamp", "Equal", "==", "timestamp", "Col", "Scalar"},
+ {"TimestampCompareTimestamp", "NotEqual", "!=", "timestamp", "Col", "Scalar"},
+ {"TimestampCompareTimestamp", "Less", "<", "timestamp", "Col", "Scalar"},
+ {"TimestampCompareTimestamp", "LessEqual", "<=", "timestamp", "Col", "Scalar"},
+ {"TimestampCompareTimestamp", "Greater", ">", "timestamp", "Col", "Scalar"},
+ {"TimestampCompareTimestamp", "GreaterEqual", ">=", "timestamp", "Col", "Scalar"},
+
+ {"TimestampCompareTimestamp", "Equal", "==", "timestamp", "Scalar", "Column"},
+ {"TimestampCompareTimestamp", "NotEqual", "!=", "timestamp", "Scalar", "Column"},
+ {"TimestampCompareTimestamp", "Less", "<", "timestamp", "Scalar", "Column"},
+ {"TimestampCompareTimestamp", "LessEqual", "<=", "timestamp", "Scalar", "Column"},
+ {"TimestampCompareTimestamp", "Greater", ">", "timestamp", "Scalar", "Column"},
+ {"TimestampCompareTimestamp", "GreaterEqual", ">=", "timestamp", "Scalar", "Column"},
+
+ {"TimestampCompareTimestamp", "Equal", "==", "interval_day_time", "Col", "Column"},
+ {"TimestampCompareTimestamp", "NotEqual", "!=", "interval_day_time", "Col", "Column"},
+ {"TimestampCompareTimestamp", "Less", "<", "interval_day_time", "Col", "Column"},
+ {"TimestampCompareTimestamp", "LessEqual", "<=", "interval_day_time", "Col", "Column"},
+ {"TimestampCompareTimestamp", "Greater", ">", "interval_day_time", "Col", "Column"},
+ {"TimestampCompareTimestamp", "GreaterEqual", ">=", "interval_day_time", "Col", "Column"},
+
+ {"TimestampCompareTimestamp", "Equal", "==", "interval_day_time", "Col", "Scalar"},
+ {"TimestampCompareTimestamp", "NotEqual", "!=", "interval_day_time", "Col", "Scalar"},
+ {"TimestampCompareTimestamp", "Less", "<", "interval_day_time", "Col", "Scalar"},
+ {"TimestampCompareTimestamp", "LessEqual", "<=", "interval_day_time", "Col", "Scalar"},
+ {"TimestampCompareTimestamp", "Greater", ">", "interval_day_time", "Col", "Scalar"},
+ {"TimestampCompareTimestamp", "GreaterEqual", ">=", "interval_day_time", "Col", "Scalar"},
+
+ {"TimestampCompareTimestamp", "Equal", "==", "interval_day_time", "Scalar", "Column"},
+ {"TimestampCompareTimestamp", "NotEqual", "!=", "interval_day_time", "Scalar", "Column"},
+ {"TimestampCompareTimestamp", "Less", "<", "interval_day_time", "Scalar", "Column"},
+ {"TimestampCompareTimestamp", "LessEqual", "<=", "interval_day_time", "Scalar", "Column"},
+ {"TimestampCompareTimestamp", "Greater", ">", "interval_day_time", "Scalar", "Column"},
+ {"TimestampCompareTimestamp", "GreaterEqual", ">=", "interval_day_time", "Scalar", "Column"},
// Compare timestamp to integer seconds or double seconds with fractional nanoseonds.
{"TimestampCompareLongDouble", "Equal", "long", "==", "Col", "Column"},
@@ -515,71 +468,49 @@ public class GenVectorCode extends Task {
{"FilterScalarCompareColumn", "GreaterEqual", "long", "long", ">="},
{"FilterScalarCompareColumn", "GreaterEqual", "double", "long", ">="},
- // Base filter timestamp against timestamp used by Timestamp and IntervalDayTime.
- {"FilterTimestampCompareTimestampBase", "Equal", "==", "Col", "Column"},
- {"FilterTimestampCompareTimestampBase", "NotEqual", "!=", "Col", "Column"},
- {"FilterTimestampCompareTimestampBase", "Less", "<", "Col", "Column"},
- {"FilterTimestampCompareTimestampBase", "LessEqual", "<=", "Col", "Column"},
- {"FilterTimestampCompareTimestampBase", "Greater", ">", "Col", "Column"},
- {"FilterTimestampCompareTimestampBase", "GreaterEqual", ">=", "Col", "Column"},
-
- {"FilterTimestampCompareTimestampBase", "Equal", "==", "Col", "Scalar"},
- {"FilterTimestampCompareTimestampBase", "NotEqual", "!=", "Col", "Scalar"},
- {"FilterTimestampCompareTimestampBase", "Less", "<", "Col", "Scalar"},
- {"FilterTimestampCompareTimestampBase", "LessEqual", "<=", "Col", "Scalar"},
- {"FilterTimestampCompareTimestampBase", "Greater", ">", "Col", "Scalar"},
- {"FilterTimestampCompareTimestampBase", "GreaterEqual", ">=", "Col", "Scalar"},
-
- {"FilterTimestampCompareTimestampBase", "Equal", "==", "Scalar", "Column"},
- {"FilterTimestampCompareTimestampBase", "NotEqual", "!=", "Scalar", "Column"},
- {"FilterTimestampCompareTimestampBase", "Less", "<", "Scalar", "Column"},
- {"FilterTimestampCompareTimestampBase", "LessEqual", "<=", "Scalar", "Column"},
- {"FilterTimestampCompareTimestampBase", "Greater", ">", "Scalar", "Column"},
- {"FilterTimestampCompareTimestampBase", "GreaterEqual", ">=", "Scalar", "Column"},
-
// Filter timestamp against timestamp, or interval day time against interval day time.
- {"FilterTimestampCompareTimestamp", "Equal", "timestamp", "Col", "Column"},
- {"FilterTimestampCompareTimestamp", "NotEqual", "timestamp", "Col", "Column"},
- {"FilterTimestampCompareTimestamp", "Less", "timestamp", "Col", "Column"},
- {"FilterTimestampCompareTimestamp", "LessEqual", "timestamp", "Col", "Column"},
- {"FilterTimestampCompareTimestamp", "Greater", "timestamp", "Col", "Column"},
- {"FilterTimestampCompareTimestamp", "GreaterEqual", "timestamp", "Col", "Column"},
-
- {"FilterTimestampCompareTimestamp", "Equal", "timestamp", "Col", "Scalar"},
- {"FilterTimestampCompareTimestamp", "NotEqual", "timestamp", "Col", "Scalar"},
- {"FilterTimestampCompareTimestamp", "Less", "timestamp", "Col", "Scalar"},
- {"FilterTimestampCompareTimestamp", "LessEqual", "timestamp", "Col", "Scalar"},
- {"FilterTimestampCompareTimestamp", "Greater", "timestamp", "Col", "Scalar"},
- {"FilterTimestampCompareTimestamp", "GreaterEqual", "timestamp", "Col", "Scalar"},
-
- {"FilterTimestampCompareTimestamp", "Equal", "timestamp", "Scalar", "Column"},
- {"FilterTimestampCompareTimestamp", "NotEqual", "timestamp", "Scalar", "Column"},
- {"FilterTimestampCompareTimestamp", "Less", "timestamp", "Scalar", "Column"},
- {"FilterTimestampCompareTimestamp", "LessEqual", "timestamp", "Scalar", "Column"},
- {"FilterTimestampCompareTimestamp", "Greater", "timestamp", "Scalar", "Column"},
- {"FilterTimestampCompareTimestamp", "GreaterEqual", "timestamp", "Scalar", "Column"},
-
- {"FilterTimestampCompareTimestamp", "Equal", "interval_day_time", "Col", "Column"},
- {"FilterTimestampCompareTimestamp", "NotEqual", "interval_day_time", "Col", "Column"},
- {"FilterTimestampCompareTimestamp", "Less", "interval_day_time", "Col", "Column"},
- {"FilterTimestampCompareTimestamp", "LessEqual", "interval_day_time", "Col", "Column"},
- {"FilterTimestampCompareTimestamp", "Greater", "interval_day_time", "Col", "Column"},
- {"FilterTimestampCompareTimestamp", "GreaterEqual", "interval_day_time", "Col", "Column"},
-
- {"FilterTimestampCompareTimestamp", "Equal", "interval_day_time", "Col", "Scalar"},
- {"FilterTimestampCompareTimestamp", "NotEqual", "interval_day_time", "Col", "Scalar"},
- {"FilterTimestampCompareTimestamp", "Less", "interval_day_time", "Col", "Scalar"},
- {"FilterTimestampCompareTimestamp", "LessEqual", "interval_day_time", "Col", "Scalar"},
- {"FilterTimestampCompareTimestamp", "Greater", "interval_day_time", "Col", "Scalar"},
- {"FilterTimestampCompareTimestamp", "GreaterEqual", "interval_day_time", "Col", "Scalar"},
-
- {"FilterTimestampCompareTimestamp", "Equal", "interval_day_time", "Scalar", "Column"},
- {"FilterTimestampCompareTimestamp", "NotEqual", "interval_day_time", "Scalar", "Column"},
- {"FilterTimestampCompareTimestamp", "Less", "interval_day_time", "Scalar", "Column"},
- {"FilterTimestampCompareTimestamp", "LessEqual", "interval_day_time", "Scalar", "Column"},
- {"FilterTimestampCompareTimestamp", "Greater", "interval_day_time", "Scalar", "Column"},
- {"FilterTimestampCompareTimestamp", "GreaterEqual", "interval_day_time", "Scalar", "Column"},
+ {"FilterTimestampCompareTimestamp", "Equal", "==", "timestamp", "Col", "Column"},
+ {"FilterTimestampCompareTimestamp", "NotEqual", "!=", "timestamp", "Col", "Column"},
+ {"FilterTimestampCompareTimestamp", "Less", "<", "timestamp", "Col", "Column"},
+ {"FilterTimestampCompareTimestamp", "LessEqual", "<=", "timestamp", "Col", "Column"},
+ {"FilterTimestampCompareTimestamp", "Greater", ">", "timestamp", "Col", "Column"},
+ {"FilterTimestampCompareTimestamp", "GreaterEqual", ">=", "timestamp", "Col", "Column"},
+
+ {"FilterTimestampCompareTimestamp", "Equal", "==", "timestamp", "Col", "Scalar"},
+ {"FilterTimestampCompareTimestamp", "NotEqual", "!=", "timestamp", "Col", "Scalar"},
+ {"FilterTimestampCompareTimestamp", "Less", "<", "timestamp", "Col", "Scalar"},
+ {"FilterTimestampCompareTimestamp", "LessEqual", "<=", "timestamp", "Col", "Scalar"},
+ {"FilterTimestampCompareTimestamp", "Greater", ">", "timestamp", "Col", "Scalar"},
+ {"FilterTimestampCompareTimestamp", "GreaterEqual", ">=", "timestamp", "Col", "Scalar"},
+
+ {"FilterTimestampCompareTimestamp", "Equal", "==", "timestamp", "Scalar", "Column"},
+ {"FilterTimestampCompareTimestamp", "NotEqual", "!=", "timestamp", "Scalar", "Column"},
+ {"FilterTimestampCompareTimestamp", "Less", "<", "timestamp", "Scalar", "Column"},
+ {"FilterTimestampCompareTimestamp", "LessEqual", "<=", "timestamp", "Scalar", "Column"},
+ {"FilterTimestampCompareTimestamp", "Greater", ">", "timestamp", "Scalar", "Column"},
+ {"FilterTimestampCompareTimestamp", "GreaterEqual", ">=", "timestamp", "Scalar", "Column"},
+
+ {"FilterTimestampCompareTimestamp", "Equal", "==", "interval_day_time", "Col", "Column"},
+ {"FilterTimestampCompareTimestamp", "NotEqual", "!=", "interval_day_time", "Col", "Column"},
+ {"FilterTimestampCompareTimestamp", "Less", "<", "interval_day_time", "Col", "Column"},
+ {"FilterTimestampCompareTimestamp", "LessEqual", "<=", "interval_day_time", "Col", "Column"},
+ {"FilterTimestampCompareTimestamp", "Greater", ">", "interval_day_time", "Col", "Column"},
+ {"FilterTimestampCompareTimestamp", "GreaterEqual", ">=", "interval_day_time", "Col", "Column"},
+
+ {"FilterTimestampCompareTimestamp", "Equal", "==", "interval_day_time", "Col", "Scalar"},
+ {"FilterTimestampCompareTimestamp", "NotEqual", "!=", "interval_day_time", "Col", "Scalar"},
+ {"FilterTimestampCompareTimestamp", "Less", "<", "interval_day_time", "Col", "Scalar"},
+ {"FilterTimestampCompareTimestamp", "LessEqual", "<=", "interval_day_time", "Col", "Scalar"},
+ {"FilterTimestampCompareTimestamp", "Greater", ">", "interval_day_time", "Col", "Scalar"},
+ {"FilterTimestampCompareTimestamp", "GreaterEqual", ">=", "interval_day_time", "Col", "Scalar"},
+
+ {"FilterTimestampCompareTimestamp", "Equal", "==", "interval_day_time", "Scalar", "Column"},
+ {"FilterTimestampCompareTimestamp", "NotEqual", "!=", "interval_day_time", "Scalar", "Column"},
+ {"FilterTimestampCompareTimestamp", "Less", "<", "interval_day_time", "Scalar", "Column"},
+ {"FilterTimestampCompareTimestamp", "LessEqual", "<=", "interval_day_time", "Scalar", "Column"},
+ {"FilterTimestampCompareTimestamp", "Greater", ">", "interval_day_time", "Scalar", "Column"},
+ {"FilterTimestampCompareTimestamp", "GreaterEqual", ">=", "interval_day_time", "Scalar", "Column"},
// Filter timestamp against long (seconds) or double (seconds with fractional
// nanoseconds).
@@ -1057,6 +988,11 @@ public class GenVectorCode extends Task {
{"VectorUDAFMinMaxTimestamp", "VectorUDAFMinTimestamp", ">", "min",
"_FUNC_(expr) - Returns the minimum value of expr (vectorized, type: timestamp)"},
+ {"VectorUDAFMinMaxIntervalDayTime", "VectorUDAFMaxIntervalDayTime", "<", "max",
+ "_FUNC_(expr) - Returns the maximum value of expr (vectorized, type: interval_day_time)"},
+ {"VectorUDAFMinMaxIntervalDayTime", "VectorUDAFMinIntervalDayTime", ">", "min",
+ "_FUNC_(expr) - Returns the minimum value of expr (vectorized, type: interval_day_time)"},
+
//template, <ClassName>, <ValueType>
{"VectorUDAFSum", "VectorUDAFSumLong", "long"},
{"VectorUDAFSum", "VectorUDAFSumDouble", "double"},
@@ -1202,9 +1138,6 @@ public class GenVectorCode extends Task {
} else if (tdesc[0].equals("ScalarCompareColumn")) {
generateScalarCompareColumn(tdesc);
- } else if (tdesc[0].equals("TimestampCompareTimestampBase")) {
- generateTimestampCompareTimestampBase(tdesc);
-
} else if (tdesc[0].equals("TimestampCompareTimestamp")) {
generateTimestampCompareTimestamp(tdesc);
@@ -1219,9 +1152,6 @@ public class GenVectorCode extends Task {
} else if (tdesc[0].equals("FilterScalarCompareColumn")) {
generateFilterScalarCompareColumn(tdesc);
- } else if (tdesc[0].equals("FilterTimestampCompareTimestampBase")) {
- generateFilterTimestampCompareTimestampBase(tdesc);
-
} else if (tdesc[0].equals("FilterTimestampCompareTimestamp")) {
generateFilterTimestampCompareTimestamp(tdesc);
@@ -1255,6 +1185,8 @@ public class GenVectorCode extends Task {
generateVectorUDAFMinMaxObject(tdesc);
} else if (tdesc[0].equals("VectorUDAFMinMaxTimestamp")) {
generateVectorUDAFMinMaxObject(tdesc);
+ } else if (tdesc[0].equals("VectorUDAFMinMaxIntervalDayTime")) {
+ generateVectorUDAFMinMaxObject(tdesc);
} else if (tdesc[0].equals("VectorUDAFSum")) {
generateVectorUDAFSum(tdesc);
} else if (tdesc[0].equals("VectorUDAFAvg")) {
@@ -1338,21 +1270,12 @@ public class GenVectorCode extends Task {
} else if (tdesc[0].equals("IntervalYearMonthArithmeticTimestamp")) {
generateDateTimeArithmeticIntervalYearMonth(tdesc);
- } else if (tdesc[0].equals("TimestampArithmeticTimestampBase")) {
- generateTimestampArithmeticTimestampBase(tdesc);
-
} else if (tdesc[0].equals("TimestampArithmeticTimestamp")) {
generateTimestampArithmeticTimestamp(tdesc);
- } else if (tdesc[0].equals("DateArithmeticTimestampBase")) {
- generateDateArithmeticTimestampBase(tdesc);
-
} else if (tdesc[0].equals("DateArithmeticTimestamp")) {
generateDateArithmeticTimestamp(tdesc);
- } else if (tdesc[0].equals("TimestampArithmeticDateBase")) {
- generateTimestampArithmeticDateBase(tdesc);
-
} else if (tdesc[0].equals("TimestampArithmeticDate")) {
generateTimestampArithmeticDate(tdesc);
@@ -2182,35 +2105,28 @@ public class GenVectorCode extends Task {
//
// -----------------------------------------------------------------------------------------------
- private void generateFilterTimestampCompareTimestampBase(String[] tdesc) throws Exception {
+ private void generateFilterTimestampCompareTimestamp(String[] tdesc) throws Exception {
String operatorName = tdesc[1];
String operatorSymbol = tdesc[2];
- String className = "FilterTimestamp" + tdesc[3] + operatorName + "Timestamp" + tdesc[4] + "Base";
+ String operandType = tdesc[3];
+ String camelOperandType = getCamelCaseType(operandType);
+ String className = "Filter" + camelOperandType + tdesc[4] + operatorName + camelOperandType + tdesc[5];
+ String baseClassName = "FilterTimestamp" + tdesc[4] + operatorName + "Timestamp" + tdesc[5] + "Base";
//Read the template into a string;
- String fileName = "FilterTimestamp" + (tdesc[3].equals("Col") ? "Column" : tdesc[3]) + "CompareTimestamp" +
- tdesc[4] + "Base";
+ String fileName = "FilterTimestamp" + (tdesc[4].equals("Col") ? "Column" : tdesc[4]) + "CompareTimestamp" +
+ tdesc[5];
File templateFile = new File(joinPath(this.expressionTemplateDirectory, fileName + ".txt"));
String templateString = readFile(templateFile);
templateString = templateString.replaceAll("<ClassName>", className);
templateString = templateString.replaceAll("<OperatorSymbol>", operatorSymbol);
- writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
- className, templateString);
- }
+ templateString = templateString.replaceAll("<OperandType>", operandType);
+ templateString = templateString.replaceAll("<CamelOperandType>", camelOperandType);
+ templateString = templateString.replaceAll("<HiveOperandType>", getTimestampHiveType(operandType));
+
+ String inputColumnVectorType = this.getColumnVectorType(operandType);
+ templateString = templateString.replaceAll("<InputColumnVectorType>", inputColumnVectorType);
- private void generateFilterTimestampCompareTimestamp(String[] tdesc) throws Exception {
- String operatorName = tdesc[1];
- String operandType = tdesc[2];
- String camelCaseOperandType = getCamelCaseType(operandType);
- String className = "Filter" + camelCaseOperandType + tdesc[3] + operatorName + camelCaseOperandType + tdesc[4];
- String baseClassName = "FilterTimestamp" + tdesc[3] + operatorName + "Timestamp" + tdesc[4] + "Base";
- //Read the template into a string;
- String fileName = "Filter" + camelCaseOperandType + (tdesc[3].equals("Col") ? "Column" : tdesc[3]) + "Compare" + camelCaseOperandType +
- tdesc[4];
- File templateFile = new File(joinPath(this.expressionTemplateDirectory, fileName + ".txt"));
- String templateString = readFile(templateFile);
- templateString = templateString.replaceAll("<ClassName>", className);
- templateString = templateString.replaceAll("<BaseClassName>", baseClassName);
writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
className, templateString);
}
@@ -2287,9 +2203,9 @@ public class GenVectorCode extends Task {
private String timestampLongDoubleMethod(String operandType) {
if (operandType.equals("long")) {
- return "getTimestampSeconds";
+ return "getTimestampAsLong";
} else if (operandType.equals("double")) {
- return "getTimestampSecondsWithFractionalNanos";
+ return "getDouble";
} else {
return "unknown";
}
@@ -2314,35 +2230,26 @@ public class GenVectorCode extends Task {
//
// -----------------------------------------------------------------------------------------------
- private void generateTimestampCompareTimestampBase(String[] tdesc) throws Exception {
+
+ private void generateTimestampCompareTimestamp(String[] tdesc) throws Exception {
String operatorName = tdesc[1];
String operatorSymbol = tdesc[2];
- String className = "Timestamp" + tdesc[3] + operatorName + "Timestamp" + tdesc[4] + "Base";
+ String operandType = tdesc[3];
+ String camelOperandType = getCamelCaseType(operandType);
+ String className = camelOperandType + tdesc[4] + operatorName + camelOperandType + tdesc[5];
//Read the template into a string;
- String fileName = "Timestamp" + (tdesc[3].equals("Col") ? "Column" : tdesc[3]) + "CompareTimestamp" +
- tdesc[4] + "Base";
+ String fileName = "Timestamp" + (tdesc[4].equals("Col") ? "Column" : tdesc[4]) + "CompareTimestamp" +
+ (tdesc[5].equals("Col") ? "Column" : tdesc[5]);
File templateFile = new File(joinPath(this.expressionTemplateDirectory, fileName + ".txt"));
String templateString = readFile(templateFile);
templateString = templateString.replaceAll("<ClassName>", className);
templateString = templateString.replaceAll("<OperatorSymbol>", operatorSymbol);
- writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
- className, templateString);
- }
+ templateString = templateString.replaceAll("<OperandType>", operandType);
+ templateString = templateString.replaceAll("<CamelOperandType>", camelOperandType);
+ templateString = templateString.replaceAll("<HiveOperandType>", getTimestampHiveType(operandType));
+ templateString = templateString.replaceAll("<InputColumnVectorType>", getColumnVectorType(operandType));
- private void generateTimestampCompareTimestamp(String[] tdesc) throws Exception {
- String operatorName = tdesc[1];
- String operandType = tdesc[2];
- String camelCaseOperandType = getCamelCaseType(operandType);
- String className = camelCaseOperandType + tdesc[3] + operatorName + camelCaseOperandType + tdesc[4];
- String baseClassName = "Timestamp" + tdesc[3] + operatorName + "Timestamp" + tdesc[4] + "Base";
- //Read the template into a string;
- String fileName = camelCaseOperandType + (tdesc[3].equals("Col") ? "Column" : tdesc[3]) + "Compare" + camelCaseOperandType +
- tdesc[4];
- File templateFile = new File(joinPath(this.expressionTemplateDirectory, fileName + ".txt"));
- String templateString = readFile(templateFile);
- templateString = templateString.replaceAll("<ClassName>", className);
- templateString = templateString.replaceAll("<BaseClassName>", baseClassName);
writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
className, templateString);
}
@@ -2851,6 +2758,7 @@ public class GenVectorCode extends Task {
String templateString = readFile(templateFile);
templateString = templateString.replaceAll("<ClassName>", className);
templateString = templateString.replaceAll("<OperatorSymbol>", operatorSymbol);
+ templateString = templateString.replaceAll("<OperatorMethod>", operatorName.toLowerCase());
writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
className, templateString);
@@ -2924,30 +2832,6 @@ public class GenVectorCode extends Task {
return templateString;
}
- // TimestampColumnArithmeticTimestampColumnBase.txt
- // TimestampScalarArithmeticTimestampColumnBase.txt
- // TimestampColumnArithmeticTimestampScalarBase.txt
- //
- private void generateTimestampArithmeticTimestampBase(String[] tdesc) throws Exception {
- String operatorName = tdesc[1];
- String colOrScalar1 = tdesc[2];
- String colOrScalar2 = tdesc[3];
-
- String baseClassName = "Timestamp" + colOrScalar1 + operatorName +
- "Timestamp" + colOrScalar2 + "Base";
-
- //Read the template into a string;
- String fileName = "Timestamp" + (colOrScalar1.equals("Col") ? "Column" : colOrScalar1) + "Arithmetic" +
- "Timestamp" + colOrScalar2 + "Base";
- File templateFile = new File(joinPath(this.expressionTemplateDirectory, fileName + ".txt"));
- String templateString = readFile(templateFile);
- templateString = templateString.replaceAll("<BaseClassName>", baseClassName);
- templateString = templateString.replaceAll("<OperatorMethod>", operatorName.toLowerCase());
-
- writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
- baseClassName, templateString);
- }
-
// TimestampColumnArithmeticTimestampColumn.txt
// TimestampScalarArithmeticTimestampColumn.txt
// TimestampColumnArithmeticTimestampScalar.txt
@@ -2955,10 +2839,23 @@ public class GenVectorCode extends Task {
private void generateTimestampArithmeticTimestamp(String[] tdesc) throws Exception {
String operatorName = tdesc[1];
String operandType1 = tdesc[2];
+ String camelOperandType1 = getCamelCaseType(operandType1);
String colOrScalar1 = tdesc[3];
String operandType2 = tdesc[4];
+ String camelOperandType2 = getCamelCaseType(operandType2);
String colOrScalar2 = tdesc[5];
+ String returnType;
+ if (operandType1.equals(operandType2)) {
+ // timestamp - timestamp
+ // interval_day_time +/- interval_day_time
+ returnType = "interval_day_time";
+ } else {
+ // timestamp +/- interval_day_time
+ // interval_day_time + timestamp
+ returnType = "timestamp";
+ }
+
String className = getCamelCaseType(operandType1) + colOrScalar1 + operatorName +
getCamelCaseType(operandType2) + colOrScalar2;
String baseClassName = "Timestamp" + colOrScalar1 + operatorName +
@@ -2971,20 +2868,26 @@ public class GenVectorCode extends Task {
String templateString = readFile(templateFile);
templateString = templateString.replaceAll("<ClassName>", className);
templateString = templateString.replaceAll("<BaseClassName>", baseClassName);
+ templateString = templateString.replaceAll("<OperatorMethod>", operatorName.toLowerCase());
templateString = templateString.replaceAll("<OperandType1>", operandType1);
templateString = templateString.replaceAll("<OperandType2>", operandType2);
- if (colOrScalar1.equals("Scalar")) {
- templateString = replaceTimestampScalar(templateString, 1, operandType1);
- }
- if (colOrScalar2.equals("Scalar")) {
- templateString = replaceTimestampScalar(templateString, 2, operandType2);
- }
-
- writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
- className, templateString);
+ templateString = templateString.replaceAll("<CamelOperandType1>", camelOperandType1);
+ templateString = templateString.replaceAll("<CamelOperandType2>", camelOperandType2);
+ templateString = templateString.replaceAll("<HiveOperandType1>", getTimestampHiveType(operandType1));
+ templateString = templateString.replaceAll("<HiveOperandType2>", getTimestampHiveType(operandType2));
String inputColumnVectorType1 = this.getColumnVectorType(operandType1);
+ templateString = templateString.replaceAll("<InputColumnVectorType1>", inputColumnVectorType1);
String inputColumnVectorType2 = this.getColumnVectorType(operandType2);
+ templateString = templateString.replaceAll("<InputColumnVectorType2>", inputColumnVectorType2);
+
+ String outputColumnVectorType = this.getColumnVectorType(returnType);
+ templateString = templateString.replaceAll("<OutputColumnVectorType>", outputColumnVectorType);
+ templateString = templateString.replaceAll("<CamelReturnType>", getCamelCaseType(returnType));
+ templateString = templateString.replaceAll("<ReturnType>", returnType);
+
+ writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
+ className, templateString);
/* UNDONE: Col Col, vs Scalar Col vs Col Scalar
testCodeGen.addColumnColumnOperationTestCases(
@@ -2995,30 +2898,6 @@ public class GenVectorCode extends Task {
*/
}
- // DateColumnArithmeticTimestampColumnBase.txt
- // DateScalarArithmeticTimestampColumnBase.txt
- // DateColumnArithmeticTimestampScalarBase.txt
- //
- private void generateDateArithmeticTimestampBase(String[] tdesc) throws Exception {
- String operatorName = tdesc[1];
- String colOrScalar1 = tdesc[2];
- String colOrScalar2 = tdesc[3];
-
- String baseClassName = "Date" + colOrScalar1 + operatorName +
- "Timestamp" + colOrScalar2 + "Base";
-
- //Read the template into a string;
- String fileName = "Date" + (colOrScalar1.equals("Col") ? "Column" : colOrScalar1) + "Arithmetic" +
- "Timestamp" + colOrScalar2 + "Base";
- File templateFile = new File(joinPath(this.expressionTemplateDirectory, fileName + ".txt"));
- String templateString = readFile(templateFile);
- templateString = templateString.replaceAll("<BaseClassName>", baseClassName);
- templateString = templateString.replaceAll("<OperatorMethod>", operatorName.toLowerCase());
-
- writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
- baseClassName, templateString);
- }
-
// DateColumnArithmeticTimestampColumn.txt
// DateScalarArithmeticTimestampColumn.txt
// DateColumnArithmeticTimestampScalar.txt
@@ -3026,14 +2905,23 @@ public class GenVectorCode extends Task {
private void generateDateArithmeticTimestamp(String[] tdesc) throws Exception {
String operatorName = tdesc[1];
String operandType1 = tdesc[2];
+ String camelOperandType1 = getCamelCaseType(operandType1);
String colOrScalar1 = tdesc[3];
String operandType2 = tdesc[4];
+ String camelOperandType2 = getCamelCaseType(operandType2);
String colOrScalar2 = tdesc[5];
- String className = getCamelCaseType(operandType1) + colOrScalar1 + operatorName +
- getCamelCaseType(operandType2) + colOrScalar2;
- String baseClassName = "Date" + colOrScalar1 + operatorName +
- "Timestamp" + colOrScalar2 + "Base";
+ String returnType;
+ if (operandType1.equals("interval_day_time") || operandType2.equals("interval_day_time")) {
+ returnType = "timestamp";
+ } else if (operandType1.equals("timestamp") || operandType2.equals("timestamp")) {
+ returnType = "interval_day_time";
+ } else {
+ returnType = "unknown";
+ }
+
+ String className = camelOperandType1 + colOrScalar1 + operatorName +
+ camelOperandType2 + colOrScalar2;
//Read the template into a string;
String fileName = "Date" + (colOrScalar1.equals("Col") ? "Column" : colOrScalar1) + "Arithmetic" +
@@ -3041,21 +2929,26 @@ public class GenVectorCode extends Task {
File templateFile = new File(joinPath(this.expressionTemplateDirectory, fileName + ".txt"));
String templateString = readFile(templateFile);
templateString = templateString.replaceAll("<ClassName>", className);
- templateString = templateString.replaceAll("<BaseClassName>", baseClassName);
+ templateString = templateString.replaceAll("<OperatorMethod>", operatorName.toLowerCase());
templateString = templateString.replaceAll("<OperandType1>", operandType1);
templateString = templateString.replaceAll("<OperandType2>", operandType2);
- if (colOrScalar1.equals("Scalar")) {
- templateString = replaceTimestampScalar(templateString, 1, operandType1);
- }
- if (colOrScalar2.equals("Scalar")) {
- templateString = replaceTimestampScalar(templateString, 2, operandType2);
- }
-
- writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
- className, templateString);
+ templateString = templateString.replaceAll("<CamelOperandType1>", camelOperandType1);
+ templateString = templateString.replaceAll("<CamelOperandType2>", camelOperandType2);
+ templateString = templateString.replaceAll("<HiveOperandType1>", getTimestampHiveType(operandType1));
+ templateString = templateString.replaceAll("<HiveOperandType2>", getTimestampHiveType(operandType2));
String inputColumnVectorType1 = this.getColumnVectorType(operandType1);
+ templateString = templateString.replaceAll("<InputColumnVectorType1>", inputColumnVectorType1);
String inputColumnVectorType2 = this.getColumnVectorType(operandType2);
+ templateString = templateString.replaceAll("<InputColumnVectorType2>", inputColumnVectorType2);
+
+ String outputColumnVectorType = this.getColumnVectorType(returnType);
+ templateString = templateString.replaceAll("<OutputColumnVectorType>", outputColumnVectorType);
+ templateString = templateString.replaceAll("<CamelReturnType>", getCamelCaseType(returnType));
+ templateString = templateString.replaceAll("<ReturnType>", returnType);
+
+ writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
+ className, templateString);
/* UNDONE: Col Col, vs Scalar Col vs Col Scalar
testCodeGen.addColumnColumnOperationTestCases(
@@ -3066,30 +2959,6 @@ public class GenVectorCode extends Task {
*/
}
- // TimestampColumnArithmeticDateColumnBase.txt
- // TimestampScalarArithmeticDateColumnBase.txt
- // TimestampColumnArithmeticDateScalarBase.txt
- //
- private void generateTimestampArithmeticDateBase(String[] tdesc) throws Exception {
- String operatorName = tdesc[1];
- String colOrScalar1 = tdesc[2];
- String colOrScalar2 = tdesc[3];
-
- String baseClassName = "Timestamp" + colOrScalar1 + operatorName +
- "Date" + colOrScalar2 + "Base";
-
- //Read the template into a string;
- String fileName = "Timestamp" + (colOrScalar1.equals("Col") ? "Column" : colOrScalar1) + "Arithmetic" +
- "Date" + colOrScalar2 + "Base";
- File templateFile = new File(joinPath(this.expressionTemplateDirectory, fileName + ".txt"));
- String templateString = readFile(templateFile);
- templateString = templateString.replaceAll("<BaseClassName>", baseClassName);
- templateString = templateString.replaceAll("<OperatorMethod>", operatorName.toLowerCase());
-
- writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
- baseClassName, templateString);
- }
-
// TimestampColumnArithmeticDateColumn.txt
// TimestampScalarArithmeticDateColumn.txt
// TimestampColumnArithmeticDateScalar.txt
@@ -3097,14 +2966,23 @@ public class GenVectorCode extends Task {
private void generateTimestampArithmeticDate(String[] tdesc) throws Exception {
String operatorName = tdesc[1];
String operandType1 = tdesc[2];
+ String camelOperandType1 = getCamelCaseType(operandType1);
String colOrScalar1 = tdesc[3];
String operandType2 = tdesc[4];
+ String camelOperandType2 = getCamelCaseType(operandType2);
String colOrScalar2 = tdesc[5];
- String className = getCamelCaseType(operandType1) + colOrScalar1 + operatorName +
- getCamelCaseType(operandType2) + colOrScalar2;
- String baseClassName = "Timestamp" + colOrScalar1 + operatorName +
- "Date" + colOrScalar2 + "Base";
+ String returnType;
+ if (operandType1.equals("interval_day_time") || operandType2.equals("interval_day_time")) {
+ returnType = "timestamp";
+ } else if (operandType1.equals("timestamp") || operandType2.equals("timestamp")) {
+ returnType = "interval_day_time";
+ } else {
+ returnType = "unknown";
+ }
+
+ String className = camelOperandType1 + colOrScalar1 + operatorName +
+ camelOperandType2 + colOrScalar2;
//Read the template into a string;
String fileName = "Timestamp" + (colOrScalar1.equals("Col") ? "Column" : colOrScalar1) + "Arithmetic" +
@@ -3112,21 +2990,26 @@ public class GenVectorCode extends Task {
File templateFile = new File(joinPath(this.expressionTemplateDirectory, fileName + ".txt"));
String templateString = readFile(templateFile);
templateString = templateString.replaceAll("<ClassName>", className);
- templateString = templateString.replaceAll("<BaseClassName>", baseClassName);
+ templateString = templateString.replaceAll("<OperatorMethod>", operatorName.toLowerCase());
templateString = templateString.replaceAll("<OperandType1>", operandType1);
templateString = templateString.replaceAll("<OperandType2>", operandType2);
- if (colOrScalar1.equals("Scalar")) {
- templateString = replaceTimestampScalar(templateString, 1, operandType1);
- }
- if (colOrScalar2.equals("Scalar")) {
- templateString = replaceTimestampScalar(templateString, 2, operandType2);
- }
-
- writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
- className, templateString);
+ templateString = templateString.replaceAll("<CamelOperandType1>", camelOperandType1);
+ templateString = templateString.replaceAll("<CamelOperandType2>", camelOperandType2);
+ templateString = templateString.replaceAll("<HiveOperandType1>", getTimestampHiveType(operandType1));
+ templateString = templateString.replaceAll("<HiveOperandType2>", getTimestampHiveType(operandType2));
String inputColumnVectorType1 = this.getColumnVectorType(operandType1);
+ templateString = templateString.replaceAll("<InputColumnVectorType1>", inputColumnVectorType1);
String inputColumnVectorType2 = this.getColumnVectorType(operandType2);
+ templateString = templateString.replaceAll("<InputColumnVectorType2>", inputColumnVectorType2);
+
+ String outputColumnVectorType = this.getColumnVectorType(returnType);
+ templateString = templateString.replaceAll("<OutputColumnVectorType>", outputColumnVectorType);
+ templateString = templateString.replaceAll("<CamelReturnType>", getCamelCaseType(returnType));
+ templateString = templateString.replaceAll("<ReturnType>", returnType);
+
+ writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
+ className, templateString);
/* UNDONE: Col Col, vs Scalar Col vs Col Scalar
testCodeGen.addColumnColumnOperationTestCases(
@@ -3272,8 +3155,10 @@ public class GenVectorCode extends Task {
return "DecimalColumnVector";
} else if (primitiveType.equals("string")) {
return "BytesColumnVector";
- } else if (isTimestampIntervalType(primitiveType)) {
+ } else if (primitiveType.equals("timestamp")) {
return "TimestampColumnVector";
+ } else if (primitiveType.equals("interval_day_time")) {
+ return "IntervalDayTimeColumnVector";
}
throw new Exception("Unimplemented primitive column vector type: " + primitiveType);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/common/src/java/org/apache/hadoop/hive/common/type/HiveIntervalDayTime.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/type/HiveIntervalDayTime.java b/common/src/java/org/apache/hadoop/hive/common/type/HiveIntervalDayTime.java
deleted file mode 100644
index e262f01..0000000
--- a/common/src/java/org/apache/hadoop/hive/common/type/HiveIntervalDayTime.java
+++ /dev/null
@@ -1,245 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.common.type;
-
-import java.math.BigDecimal;
-import java.util.concurrent.TimeUnit;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.hive.common.util.DateUtils;
-
-/**
- * Day-time interval type representing an offset in days/hours/minutes/seconds,
- * with nanosecond precision.
- * 1 day = 24 hours = 1440 minutes = 86400 seconds
- */
-public class HiveIntervalDayTime implements Comparable<HiveIntervalDayTime> {
-
- // days/hours/minutes/seconds all represented as seconds
- protected long totalSeconds;
- protected int nanos;
-
- public HiveIntervalDayTime() {
- }
-
- public HiveIntervalDayTime(int days, int hours, int minutes, int seconds, int nanos) {
- set(days, hours, minutes, seconds, nanos);
- }
-
- public HiveIntervalDayTime(long seconds, int nanos) {
- set(seconds, nanos);
- }
-
- public HiveIntervalDayTime(BigDecimal seconds) {
- set(seconds);
- }
-
- public HiveIntervalDayTime(HiveIntervalDayTime other) {
- set(other.totalSeconds, other.nanos);
- }
-
- public int getDays() {
- return (int) TimeUnit.SECONDS.toDays(totalSeconds);
- }
-
- public int getHours() {
- return (int) (TimeUnit.SECONDS.toHours(totalSeconds) % TimeUnit.DAYS.toHours(1));
- }
-
- public int getMinutes() {
- return (int) (TimeUnit.SECONDS.toMinutes(totalSeconds) % TimeUnit.HOURS.toMinutes(1));
- }
-
- public int getSeconds() {
- return (int) (totalSeconds % TimeUnit.MINUTES.toSeconds(1));
- }
-
- public int getNanos() {
- return nanos;
- }
-
- /**
- * Returns days/hours/minutes all converted into seconds.
- * Nanos still need to be retrieved using getNanos()
- * @return
- */
- public long getTotalSeconds() {
- return totalSeconds;
- }
-
- /**
- * Ensures that the seconds and nanoseconds fields have consistent sign
- */
- protected void normalizeSecondsAndNanos() {
- if (totalSeconds > 0 && nanos < 0) {
- --totalSeconds;
- nanos += DateUtils.NANOS_PER_SEC;
- } else if (totalSeconds < 0 && nanos > 0) {
- ++totalSeconds;
- nanos -= DateUtils.NANOS_PER_SEC;
- }
- }
-
- public void set(int days, int hours, int minutes, int seconds, int nanos) {
- long totalSeconds = seconds;
- totalSeconds += TimeUnit.DAYS.toSeconds(days);
- totalSeconds += TimeUnit.HOURS.toSeconds(hours);
- totalSeconds += TimeUnit.MINUTES.toSeconds(minutes);
- totalSeconds += TimeUnit.NANOSECONDS.toSeconds(nanos);
- nanos = nanos % DateUtils.NANOS_PER_SEC;
-
- this.totalSeconds = totalSeconds;
- this.nanos = nanos;
-
- normalizeSecondsAndNanos();
- }
-
- public void set(long seconds, int nanos) {
- this.totalSeconds = seconds;
- this.nanos = nanos;
- normalizeSecondsAndNanos();
- }
-
- public void set(PisaTimestamp pisaTimestamp) {
- this.totalSeconds = pisaTimestamp.getEpochSeconds();
- this.nanos = pisaTimestamp.getSignedNanos();
- normalizeSecondsAndNanos();
- }
-
- public void set(BigDecimal totalSecondsBd) {
- long totalSeconds = totalSecondsBd.longValue();
- BigDecimal fractionalSecs = totalSecondsBd.remainder(BigDecimal.ONE);
- int nanos = fractionalSecs.multiply(DateUtils.NANOS_PER_SEC_BD).intValue();
- set(totalSeconds, nanos);
- }
-
- public void set(HiveIntervalDayTime other) {
- set(other.getTotalSeconds(), other.getNanos());
- }
-
- public HiveIntervalDayTime negate() {
- return new HiveIntervalDayTime(-getTotalSeconds(), -getNanos());
- }
-
- public PisaTimestamp pisaTimestampUpdate(PisaTimestamp pisaTimestamp) {
- // NOTE: Our nanos here are *SIGNED*.
- return pisaTimestamp.updateFromEpochSecondsAndSignedNanos(totalSeconds, nanos);
- }
-
- @Override
- public int compareTo(HiveIntervalDayTime other) {
- long cmp = this.totalSeconds - other.totalSeconds;
- if (cmp == 0) {
- cmp = this.nanos - other.nanos;
- }
- if (cmp != 0) {
- cmp = cmp > 0 ? 1 : -1;
- }
- return (int) cmp;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj) {
- return true;
- }
- if (!(obj instanceof HiveIntervalDayTime)) {
- return false;
- }
- return 0 == compareTo((HiveIntervalDayTime) obj);
- }
-
- @Override
- public int hashCode() {
- return new HashCodeBuilder().append(totalSeconds).append(nanos).toHashCode();
- }
-
- @Override
- public String toString() {
- // If normalize() was used, then day-hour-minute-second-nanos should have the same sign.
- // This is currently working with that assumption.
- boolean isNegative = (totalSeconds < 0 || nanos < 0);
- String daySecondSignStr = isNegative ? "-" : "";
-
- return String.format("%s%d %02d:%02d:%02d.%09d",
- daySecondSignStr, Math.abs(getDays()),
- Math.abs(getHours()), Math.abs(getMinutes()),
- Math.abs(getSeconds()), Math.abs(getNanos()));
- }
-
- public static HiveIntervalDayTime valueOf(String strVal) {
- HiveIntervalDayTime result = null;
- if (strVal == null) {
- throw new IllegalArgumentException("Interval day-time string was null");
- }
- Matcher patternMatcher = PATTERN_MATCHER.get();
- patternMatcher.reset(strVal);
- if (patternMatcher.matches()) {
- // Parse out the individual parts
- try {
- // Sign - whether interval is positive or negative
- int sign = 1;
- String field = patternMatcher.group(1);
- if (field != null && field.equals("-")) {
- sign = -1;
- }
- int days = sign *
- DateUtils.parseNumericValueWithRange("day", patternMatcher.group(2),
- 0, Integer.MAX_VALUE);
- byte hours = (byte) (sign *
- DateUtils.parseNumericValueWithRange("hour", patternMatcher.group(3), 0, 23));
- byte minutes = (byte) (sign *
- DateUtils.parseNumericValueWithRange("minute", patternMatcher.group(4), 0, 59));
- int seconds = 0;
- int nanos = 0;
- field = patternMatcher.group(5);
- if (field != null) {
- BigDecimal bdSeconds = new BigDecimal(field);
- if (bdSeconds.compareTo(DateUtils.MAX_INT_BD) > 0) {
- throw new IllegalArgumentException("seconds value of " + bdSeconds + " too large");
- }
- seconds = sign * bdSeconds.intValue();
- nanos = sign * bdSeconds.subtract(new BigDecimal(bdSeconds.toBigInteger()))
- .multiply(DateUtils.NANOS_PER_SEC_BD).intValue();
- }
-
- result = new HiveIntervalDayTime(days, hours, minutes, seconds, nanos);
- } catch (Exception err) {
- throw new IllegalArgumentException("Error parsing interval day-time string: " + strVal, err);
- }
- } else {
- throw new IllegalArgumentException(
- "Interval string does not match day-time format of 'd h:m:s.n': " + strVal);
- }
-
- return result;
- }
-
- // Simple pattern: D H:M:S.nnnnnnnnn
- private final static String PARSE_PATTERN =
- "([+|-])?(\\d+) (\\d+):(\\d+):((\\d+)(\\.(\\d+))?)";
-
- private static final ThreadLocal<Matcher> PATTERN_MATCHER = new ThreadLocal<Matcher>() {
- @Override
- protected Matcher initialValue() {
- return Pattern.compile(PARSE_PATTERN).matcher("");
- }
- };
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/common/src/java/org/apache/hive/common/util/DateUtils.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hive/common/util/DateUtils.java b/common/src/java/org/apache/hive/common/util/DateUtils.java
index c749bcb..959a542 100644
--- a/common/src/java/org/apache/hive/common/util/DateUtils.java
+++ b/common/src/java/org/apache/hive/common/util/DateUtils.java
@@ -21,8 +21,6 @@ package org.apache.hive.common.util;
import java.math.BigDecimal;
import java.text.SimpleDateFormat;
-import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
-
/**
* DateUtils. Thread-safe class
*
@@ -56,21 +54,4 @@ public class DateUtils {
}
return result;
}
-
- public static long getIntervalDayTimeTotalNanos(HiveIntervalDayTime intervalDayTime) {
- return intervalDayTime.getTotalSeconds() * NANOS_PER_SEC + intervalDayTime.getNanos();
- }
-
- public static void setIntervalDayTimeTotalNanos(HiveIntervalDayTime intervalDayTime,
- long totalNanos) {
- intervalDayTime.set(totalNanos / NANOS_PER_SEC, (int) (totalNanos % NANOS_PER_SEC));
- }
-
- public static long getIntervalDayTimeTotalSecondsFromTotalNanos(long totalNanos) {
- return totalNanos / NANOS_PER_SEC;
- }
-
- public static int getIntervalDayTimeNanosFromTotalNanos(long totalNanos) {
- return (int) (totalNanos % NANOS_PER_SEC);
- }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/data/files/timestamps.txt
----------------------------------------------------------------------
diff --git a/data/files/timestamps.txt b/data/files/timestamps.txt
new file mode 100644
index 0000000..36ffd23
--- /dev/null
+++ b/data/files/timestamps.txt
@@ -0,0 +1,50 @@
+6631-11-13 16:31:29.702202248
+6731-02-12 08:12:48.287783702
+6705-09-28 18:27:28.000845672
+5397-07-13 07:12:32.000896438
+9209-11-11 04:08:58.223768453
+9403-01-09 18:12:33.547
+6482-04-27 12:07:38.073915413
+7503-06-23 23:14:17.486
+1883-04-17 04:14:34.647766229
+0004-09-22 18:26:29.519542222
+7160-12-02 06:00:24.81200852
+8422-07-22 03:21:45.745036084
+4143-07-08 10:53:27.252802259
+5344-10-04 18:40:08.165
+5966-07-09 03:30:50.597
+9075-06-13 16:20:09.218517797
+1815-05-06 00:12:37.543584705
+7409-09-07 23:33:32.459349602
+5339-02-01 14:10:01.085678691
+4966-12-04 09:30:55.202
+1319-02-02 16:31:57.778
+1404-07-23 15:32:16.059185026
+6229-06-28 02:54:28.970117179
+0528-10-27 08:15:18.941718273
+8521-01-16 20:42:05.668832388
+1976-05-06 00:42:30.910786948
+2003-09-23 22:33:17.00003252
+2007-02-09 05:17:29.368756876
+1998-10-16 20:05:29.397591987
+1976-03-03 04:54:33.000895162
+1985-07-20 09:30:11.0
+2021-09-24 03:18:32.413655165
+2013-04-07 02:44:43.00086821
+2002-05-10 05:29:48.990818073
+1973-04-17 06:30:38.596784156
+1987-02-21 19:48:29.0
+1981-11-15 23:03:10.999338387
+2000-12-18 08:42:30.000595596
+1999-10-03 16:59:10.396903939
+2024-11-11 16:42:41.101
+2013-04-10 00:43:46.854731546
+2010-04-08 02:43:35.861742727
+2004-03-07 20:14:13.0
+1987-05-28 13:52:07.900916635
+1978-08-05 14:41:05.501
+1966-08-16 13:36:50.183618031
+2009-01-21 10:49:07.108
+1981-04-25 09:01:12.077192689
+1985-11-18 16:37:54.0
+1974-10-04 17:21:03.989
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index f8e8bda..0672e0e 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -289,6 +289,7 @@ minitez.query.files.shared=acid_globallimit.q,\
vector_inner_join.q,\
vector_interval_1.q,\
vector_interval_2.q,\
+ vector_interval_arithmetic.q,\
vector_interval_mapjoin.q,\
vector_join30.q,\
vector_join_filters.q,\
@@ -319,6 +320,7 @@ minitez.query.files.shared=acid_globallimit.q,\
vector_reduce3.q,\
vector_string_concat.q,\
vector_struct_in.q,\
+ vectorized_timestamp.q,\
vector_varchar_4.q,\
vector_varchar_mapjoin1.q,\
vector_varchar_simple.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/orc/src/java/org/apache/orc/impl/WriterImpl.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/impl/WriterImpl.java b/orc/src/java/org/apache/orc/impl/WriterImpl.java
index 6497ecf..d4b9a14 100644
--- a/orc/src/java/org/apache/orc/impl/WriterImpl.java
+++ b/orc/src/java/org/apache/orc/impl/WriterImpl.java
@@ -1737,19 +1737,17 @@ public class WriterImpl implements Writer, MemoryManager.Callback {
int length) throws IOException {
super.writeBatch(vector, offset, length);
TimestampColumnVector vec = (TimestampColumnVector) vector;
+ Timestamp val;
if (vector.isRepeating) {
if (vector.noNulls || !vector.isNull[0]) {
- long millis = vec.getEpochMilliseconds(0);
- int adjustedNanos = vec.getSignedNanos(0);
- if (adjustedNanos < 0) {
- adjustedNanos += NANOS_PER_SECOND;
- }
+ val = vec.asScratchTimestamp(0);
+ long millis = val.getTime();
indexStatistics.updateTimestamp(millis);
if (createBloomFilter) {
bloomFilter.addLong(millis);
}
- final long secs = vec.getEpochSeconds(0) - base_timestamp;
- final long nano = formatNanos(adjustedNanos);
+ final long secs = millis / MILLIS_PER_SECOND - base_timestamp;
+ final long nano = formatNanos(val.getNanos());
for(int i=0; i < length; ++i) {
seconds.write(secs);
nanos.write(nano);
@@ -1758,14 +1756,11 @@ public class WriterImpl implements Writer, MemoryManager.Callback {
} else {
for(int i=0; i < length; ++i) {
if (vec.noNulls || !vec.isNull[i + offset]) {
- long secs = vec.getEpochSeconds(i + offset) - base_timestamp;
- long millis = vec.getEpochMilliseconds(i + offset);
- int adjustedNanos = vec.getSignedNanos(i + offset);
- if (adjustedNanos < 0) {
- adjustedNanos += NANOS_PER_SECOND;
- }
+ val = vec.asScratchTimestamp(i + offset);
+ long millis = val.getTime();
+ long secs = millis / MILLIS_PER_SECOND - base_timestamp;
seconds.write(secs);
- nanos.write(formatNanos(adjustedNanos));
+ nanos.write(formatNanos(val.getNanos()));
indexStatistics.updateTimestamp(millis);
if (createBloomFilter) {
bloomFilter.addLong(millis);
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticIntervalYearMonthColumn.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticIntervalYearMonthColumn.txt b/ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticIntervalYearMonthColumn.txt
index 845bc5f..c3d8d7e 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticIntervalYearMonthColumn.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticIntervalYearMonthColumn.txt
@@ -18,15 +18,18 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
+import java.sql.Date;
+import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil;
import org.apache.hadoop.hive.ql.exec.vector.*;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
import org.apache.hadoop.hive.ql.util.DateTimeMath;
+import org.apache.hadoop.hive.serde2.io.DateWritable;
/**
- * Generated from template DateColumnArithmeticIntervalYearMonthColumn.txt, which covers binary arithmetic
+ * Generated from template DateColumnArithmeticIntervalYearMonthColumn.txt, which covers binary arithmetic
* expressions between date and interval year month columns.
*/
public class <ClassName> extends VectorExpression {
@@ -36,12 +39,18 @@ public class <ClassName> extends VectorExpression {
private int colNum1;
private int colNum2;
private int outputColumn;
+ private Date scratchDate1;
+ private HiveIntervalYearMonth scratchIntervalYearMonth2;
+ private Date outputDate;
private DateTimeMath dtm = new DateTimeMath();
public <ClassName>(int colNum1, int colNum2, int outputColumn) {
this.colNum1 = colNum1;
this.colNum2 = colNum2;
this.outputColumn = outputColumn;
+ scratchDate1 = new Date(0);
+ scratchIntervalYearMonth2 = new HiveIntervalYearMonth();
+ outputDate = new Date(0);
}
public <ClassName>() {
@@ -54,10 +63,10 @@ public class <ClassName> extends VectorExpression {
super.evaluateChildren(batch);
}
- // Input #1 is type date (epochDays).
+ // Input #1 is type date.
LongColumnVector inputColVector1 = (LongColumnVector) batch.cols[colNum1];
- // Input #2 is type interval_year_month (months).
+ // Input #2 is type interval_year_month.
LongColumnVector inputColVector2 = (LongColumnVector) batch.cols[colNum2];
// Output is type date.
@@ -89,38 +98,65 @@ public class <ClassName> extends VectorExpression {
* conditional checks in the inner loop.
*/
if (inputColVector1.isRepeating && inputColVector2.isRepeating) {
- outputVector[0] = dtm.addMonthsToDays(vector1[0], <OperatorSymbol> (int) vector2[0]);
+ scratchDate1.setTime(DateWritable.daysToMillis((int) vector1[0]));
+ scratchIntervalYearMonth2.set((int) vector2[0]);
+ dtm.<OperatorMethod>(
+ scratchDate1, scratchIntervalYearMonth2, outputDate);
+ outputVector[0] = DateWritable.dateToDays(outputDate);
} else if (inputColVector1.isRepeating) {
+ scratchDate1.setTime(DateWritable.daysToMillis((int) vector1[0]));
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputVector[i] = dtm.addMonthsToDays(vector1[0], <OperatorSymbol> (int) vector2[i]);
+ scratchIntervalYearMonth2.set((int) vector2[i]);
+ dtm.<OperatorMethod>(
+ scratchDate1, scratchIntervalYearMonth2, outputDate);
+ outputVector[i] = DateWritable.dateToDays(outputDate);
}
} else {
for(int i = 0; i != n; i++) {
- outputVector[i] = dtm.addMonthsToDays(vector1[0], <OperatorSymbol> (int) vector2[i]);
+ scratchIntervalYearMonth2.set((int) vector2[i]);
+ dtm.<OperatorMethod>(
+ scratchDate1, scratchIntervalYearMonth2, outputDate);
+ outputVector[i] = DateWritable.dateToDays(outputDate);
}
}
} else if (inputColVector2.isRepeating) {
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputVector[i] = dtm.addMonthsToDays(vector1[i], <OperatorSymbol> (int) vector2[0]);
+ scratchDate1.setTime(DateWritable.daysToMillis((int) vector1[i]));
+ scratchIntervalYearMonth2.set((int) vector2[i]);
+ dtm.<OperatorMethod>(
+ scratchDate1, scratchIntervalYearMonth2, outputDate);
+ outputVector[i] = DateWritable.dateToDays(outputDate);
}
} else {
for(int i = 0; i != n; i++) {
- outputVector[i] = dtm.addMonthsToDays(vector1[i], <OperatorSymbol> (int) vector2[0]);
+ scratchDate1.setTime(DateWritable.daysToMillis((int) vector1[i]));
+ scratchIntervalYearMonth2.set((int) vector2[i]);
+ dtm.<OperatorMethod>(
+ scratchDate1, scratchIntervalYearMonth2, outputDate);
+ outputVector[i] = DateWritable.dateToDays(outputDate);
}
}
} else {
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputVector[i] = dtm.addMonthsToDays(vector1[i], <OperatorSymbol> (int) vector2[i]);
+ scratchDate1.setTime(DateWritable.daysToMillis((int) vector1[i]));
+ scratchIntervalYearMonth2.set((int) vector2[i]);
+ dtm.<OperatorMethod>(
+ scratchDate1, scratchIntervalYearMonth2, outputDate);
+ outputVector[i] = DateWritable.dateToDays(outputDate);
}
} else {
for(int i = 0; i != n; i++) {
- outputVector[i] = dtm.addMonthsToDays(vector1[i], <OperatorSymbol> (int) vector2[i]);
+ scratchDate1.setTime(DateWritable.daysToMillis((int) vector1[i]));
+ scratchIntervalYearMonth2.set((int) vector2[i]);
+ dtm.<OperatorMethod>(
+ scratchDate1, scratchIntervalYearMonth2, outputDate);
+ outputVector[i] = DateWritable.dateToDays(outputDate);
}
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticIntervalYearMonthScalar.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticIntervalYearMonthScalar.txt b/ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticIntervalYearMonthScalar.txt
index 86a95c9..d1474fb 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticIntervalYearMonthScalar.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticIntervalYearMonthScalar.txt
@@ -18,6 +18,8 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
+import java.sql.Date;
+import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
@@ -25,6 +27,7 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
import org.apache.hadoop.hive.ql.exec.vector.*;
import org.apache.hadoop.hive.ql.util.DateTimeMath;
+import org.apache.hadoop.hive.serde2.io.DateWritable;
/**
* Generated from template DateColumnArithmeticIntervalYearMonthScalar.txt, which covers binary arithmetic
@@ -35,14 +38,18 @@ public class <ClassName> extends VectorExpression {
private static final long serialVersionUID = 1L;
private int colNum;
- private long value;
+ private HiveIntervalYearMonth value;
private int outputColumn;
+ private Date scratchDate1;
+ private Date outputDate;
private DateTimeMath dtm = new DateTimeMath();
public <ClassName>(int colNum, long value, int outputColumn) {
this.colNum = colNum;
- this.value = value;
+ this.value = new HiveIntervalYearMonth((int) value);
this.outputColumn = outputColumn;
+ scratchDate1 = new Date(0);
+ outputDate = new Date(0);
}
public <ClassName>() {
@@ -55,19 +62,19 @@ public class <ClassName> extends VectorExpression {
super.evaluateChildren(batch);
}
- // Input #1 is type date (epochDays).
- LongColumnVector inputColVector = (LongColumnVector) batch.cols[colNum];
+ // Input #1 is type date.
+ LongColumnVector inputColVector1 = (LongColumnVector) batch.cols[colNum];
// Output is type date.
LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumn];
int[] sel = batch.selected;
- boolean[] inputIsNull = inputColVector.isNull;
+ boolean[] inputIsNull = inputColVector1.isNull;
boolean[] outputIsNull = outputColVector.isNull;
- outputColVector.noNulls = inputColVector.noNulls;
- outputColVector.isRepeating = inputColVector.isRepeating;
+ outputColVector.noNulls = inputColVector1.noNulls;
+ outputColVector.isRepeating = inputColVector1.isRepeating;
int n = batch.size;
- long[] vector = inputColVector.vector;
+ long[] vector1 = inputColVector1.vector;
long[] outputVector = outputColVector.vector;
// return immediately if batch is empty
@@ -75,32 +82,46 @@ public class <ClassName> extends VectorExpression {
return;
}
- if (inputColVector.isRepeating) {
- outputVector[0] = dtm.addMonthsToDays(vector[0], <OperatorSymbol> (int) value);
-
- // Even if there are no nulls, we always copy over entry 0. Simplifies code.
+ if (inputColVector1.isRepeating) {
+ scratchDate1.setTime(DateWritable.daysToMillis((int) vector1[0]));
+ dtm.<OperatorMethod>(
+ scratchDate1, value, outputDate);
+ outputVector[0] = DateWritable.dateToDays(outputDate);
+ // Even if there are no nulls, we always copy over entry 0. Simplifies code.
outputIsNull[0] = inputIsNull[0];
- } else if (inputColVector.noNulls) {
+ } else if (inputColVector1.noNulls) {
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputVector[i] = dtm.addMonthsToDays(vector[i], <OperatorSymbol> (int) value);
+ scratchDate1.setTime(DateWritable.daysToMillis((int) vector1[i]));
+ dtm.<OperatorMethod>(
+ scratchDate1, value, outputDate);
+ outputVector[i] = DateWritable.dateToDays(outputDate);
}
} else {
for(int i = 0; i != n; i++) {
- outputVector[i] = dtm.addMonthsToDays(vector[i], <OperatorSymbol> (int) value);
+ scratchDate1.setTime(DateWritable.daysToMillis((int) vector1[i]));
+ dtm.<OperatorMethod>(
+ scratchDate1, value, outputDate);
+ outputVector[i] = DateWritable.dateToDays(outputDate);
}
}
} else /* there are nulls */ {
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputVector[i] = dtm.addMonthsToDays(vector[i], <OperatorSymbol> (int) value);
+ scratchDate1.setTime(DateWritable.daysToMillis((int) vector1[i]));
+ dtm.<OperatorMethod>(
+ scratchDate1, value, outputDate);
+ outputVector[i] = DateWritable.dateToDays(outputDate);
outputIsNull[i] = inputIsNull[i];
}
} else {
for(int i = 0; i != n; i++) {
- outputVector[i] = dtm.addMonthsToDays(vector[i], <OperatorSymbol> (int) value);
+ scratchDate1.setTime(DateWritable.daysToMillis((int) vector1[i]));
+ dtm.<OperatorMethod>(
+ scratchDate1, value, outputDate);
+ outputVector[i] = DateWritable.dateToDays(outputDate);
}
System.arraycopy(inputIsNull, 0, outputIsNull, 0, n);
}
[02/50] [abbrv] hive git commit: HIVE-13300 : Hive on spark throws
exception for multi-insert with join (Szehon, reviewed by Xuefu and Chao Sun)
Posted by jd...@apache.org.
HIVE-13300 : Hive on spark throws exception for multi-insert with join (Szehon, reviewed by Xuefu and Chao Sun)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/219d3527
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/219d3527
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/219d3527
Branch: refs/heads/llap
Commit: 219d3527cfac09045f0ac247821746e7c95dcb8c
Parents: e665f02
Author: Szehon Ho <sz...@cloudera.com>
Authored: Thu Mar 24 11:08:04 2016 -0700
Committer: Szehon Ho <sz...@cloudera.com>
Committed: Thu Mar 24 11:09:10 2016 -0700
----------------------------------------------------------------------
.../test/resources/testconfiguration.properties | 1 +
.../ql/exec/spark/SparkReduceRecordHandler.java | 2 +
.../clientpositive/multi_insert_with_join.q | 29 +++++
.../clientpositive/multi_insert_with_join.q.out | 128 +++++++++++++++++++
.../spark/multi_insert_with_join.q.out | 128 +++++++++++++++++++
5 files changed, 288 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/219d3527/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 232e262..f8e8bda 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -1006,6 +1006,7 @@ spark.query.files=add_part_multiple.q, \
multi_insert_lateral_view.q, \
multi_insert_mixed.q, \
multi_insert_move_tasks_share_dependencies.q, \
+ multi_insert_with_join.q, \
multi_join_union.q, \
multi_join_union_src.q, \
multigroupby_singlemr.q, \
http://git-wip-us.apache.org/repos/asf/hive/blob/219d3527/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java
index 439e0df..0d31e5f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java
@@ -230,8 +230,10 @@ public class SparkReduceRecordHandler extends SparkRecordHandler {
if (isTagged) {
// remove the tag from key coming out of reducer
// and store it in separate variable.
+ // make a copy for multi-insert with join case as Spark re-uses input key from same parent
int size = keyWritable.getSize() - 1;
tag = keyWritable.get()[size];
+ keyWritable = new BytesWritable(keyWritable.getBytes(), size);
keyWritable.setSize(size);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/219d3527/ql/src/test/queries/clientpositive/multi_insert_with_join.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/multi_insert_with_join.q b/ql/src/test/queries/clientpositive/multi_insert_with_join.q
new file mode 100644
index 0000000..862dd9f
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/multi_insert_with_join.q
@@ -0,0 +1,29 @@
+set hive.auto.convert.join=false;
+
+drop table if exists status_updates;
+drop table if exists profiles;
+drop table if exists school_summary;
+drop table if exists gender_summary;
+
+create table status_updates(userid int,status string,ds string);
+create table profiles(userid int,school string,gender int);
+create table school_summary(school string,cnt int) partitioned by (ds string);
+create table gender_summary(gender int, cnt int) partitioned by (ds string);
+
+insert into status_updates values (1, "status_1", "2009-03-20");
+insert into profiles values (1, "school_1", 0);
+
+FROM (SELECT a.status, b.school, b.gender
+FROM status_updates a JOIN profiles b
+ON (a.userid = b.userid and
+a.ds='2009-03-20' )
+) subq1
+INSERT OVERWRITE TABLE gender_summary
+PARTITION(ds='2009-03-20')
+SELECT subq1.gender, COUNT(1) GROUP BY subq1.gender
+INSERT OVERWRITE TABLE school_summary
+PARTITION(ds='2009-03-20')
+SELECT subq1.school, COUNT(1) GROUP BY subq1.school;
+
+select * from school_summary;
+select * from gender_summary;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/219d3527/ql/src/test/results/clientpositive/multi_insert_with_join.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/multi_insert_with_join.q.out b/ql/src/test/results/clientpositive/multi_insert_with_join.q.out
new file mode 100644
index 0000000..28bce84
--- /dev/null
+++ b/ql/src/test/results/clientpositive/multi_insert_with_join.q.out
@@ -0,0 +1,128 @@
+PREHOOK: query: drop table if exists status_updates
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists status_updates
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: drop table if exists profiles
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists profiles
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: drop table if exists school_summary
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists school_summary
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: drop table if exists gender_summary
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists gender_summary
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table status_updates(userid int,status string,ds string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@status_updates
+POSTHOOK: query: create table status_updates(userid int,status string,ds string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@status_updates
+PREHOOK: query: create table profiles(userid int,school string,gender int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@profiles
+POSTHOOK: query: create table profiles(userid int,school string,gender int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@profiles
+PREHOOK: query: create table school_summary(school string,cnt int) partitioned by (ds string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@school_summary
+POSTHOOK: query: create table school_summary(school string,cnt int) partitioned by (ds string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@school_summary
+PREHOOK: query: create table gender_summary(gender int, cnt int) partitioned by (ds string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@gender_summary
+POSTHOOK: query: create table gender_summary(gender int, cnt int) partitioned by (ds string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@gender_summary
+PREHOOK: query: insert into status_updates values (1, "status_1", "2009-03-20")
+PREHOOK: type: QUERY
+PREHOOK: Input: default@values__tmp__table__1
+PREHOOK: Output: default@status_updates
+POSTHOOK: query: insert into status_updates values (1, "status_1", "2009-03-20")
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@values__tmp__table__1
+POSTHOOK: Output: default@status_updates
+POSTHOOK: Lineage: status_updates.ds SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ]
+POSTHOOK: Lineage: status_updates.status SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+POSTHOOK: Lineage: status_updates.userid EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: insert into profiles values (1, "school_1", 0)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@values__tmp__table__2
+PREHOOK: Output: default@profiles
+POSTHOOK: query: insert into profiles values (1, "school_1", 0)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@values__tmp__table__2
+POSTHOOK: Output: default@profiles
+POSTHOOK: Lineage: profiles.gender EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col3, type:string, comment:), ]
+POSTHOOK: Lineage: profiles.school SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+POSTHOOK: Lineage: profiles.userid EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: FROM (SELECT a.status, b.school, b.gender
+FROM status_updates a JOIN profiles b
+ON (a.userid = b.userid and
+a.ds='2009-03-20' )
+) subq1
+INSERT OVERWRITE TABLE gender_summary
+PARTITION(ds='2009-03-20')
+SELECT subq1.gender, COUNT(1) GROUP BY subq1.gender
+INSERT OVERWRITE TABLE school_summary
+PARTITION(ds='2009-03-20')
+SELECT subq1.school, COUNT(1) GROUP BY subq1.school
+PREHOOK: type: QUERY
+PREHOOK: Input: default@profiles
+PREHOOK: Input: default@status_updates
+PREHOOK: Output: default@gender_summary@ds=2009-03-20
+PREHOOK: Output: default@school_summary@ds=2009-03-20
+POSTHOOK: query: FROM (SELECT a.status, b.school, b.gender
+FROM status_updates a JOIN profiles b
+ON (a.userid = b.userid and
+a.ds='2009-03-20' )
+) subq1
+INSERT OVERWRITE TABLE gender_summary
+PARTITION(ds='2009-03-20')
+SELECT subq1.gender, COUNT(1) GROUP BY subq1.gender
+INSERT OVERWRITE TABLE school_summary
+PARTITION(ds='2009-03-20')
+SELECT subq1.school, COUNT(1) GROUP BY subq1.school
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@profiles
+POSTHOOK: Input: default@status_updates
+POSTHOOK: Output: default@gender_summary@ds=2009-03-20
+POSTHOOK: Output: default@school_summary@ds=2009-03-20
+POSTHOOK: Lineage: gender_summary PARTITION(ds=2009-03-20).cnt EXPRESSION [(status_updates)a.null, (profiles)b.null, ]
+POSTHOOK: Lineage: gender_summary PARTITION(ds=2009-03-20).gender SIMPLE [(profiles)b.FieldSchema(name:gender, type:int, comment:null), ]
+POSTHOOK: Lineage: school_summary PARTITION(ds=2009-03-20).cnt EXPRESSION [(status_updates)a.null, (profiles)b.null, ]
+POSTHOOK: Lineage: school_summary PARTITION(ds=2009-03-20).school SIMPLE [(profiles)b.FieldSchema(name:school, type:string, comment:null), ]
+PREHOOK: query: select * from school_summary
+PREHOOK: type: QUERY
+PREHOOK: Input: default@school_summary
+PREHOOK: Input: default@school_summary@ds=2009-03-20
+#### A masked pattern was here ####
+POSTHOOK: query: select * from school_summary
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@school_summary
+POSTHOOK: Input: default@school_summary@ds=2009-03-20
+#### A masked pattern was here ####
+school_1 1 2009-03-20
+PREHOOK: query: select * from gender_summary
+PREHOOK: type: QUERY
+PREHOOK: Input: default@gender_summary
+PREHOOK: Input: default@gender_summary@ds=2009-03-20
+#### A masked pattern was here ####
+POSTHOOK: query: select * from gender_summary
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@gender_summary
+POSTHOOK: Input: default@gender_summary@ds=2009-03-20
+#### A masked pattern was here ####
+0 1 2009-03-20
http://git-wip-us.apache.org/repos/asf/hive/blob/219d3527/ql/src/test/results/clientpositive/spark/multi_insert_with_join.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/multi_insert_with_join.q.out b/ql/src/test/results/clientpositive/spark/multi_insert_with_join.q.out
new file mode 100644
index 0000000..28bce84
--- /dev/null
+++ b/ql/src/test/results/clientpositive/spark/multi_insert_with_join.q.out
@@ -0,0 +1,128 @@
+PREHOOK: query: drop table if exists status_updates
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists status_updates
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: drop table if exists profiles
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists profiles
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: drop table if exists school_summary
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists school_summary
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: drop table if exists gender_summary
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists gender_summary
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table status_updates(userid int,status string,ds string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@status_updates
+POSTHOOK: query: create table status_updates(userid int,status string,ds string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@status_updates
+PREHOOK: query: create table profiles(userid int,school string,gender int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@profiles
+POSTHOOK: query: create table profiles(userid int,school string,gender int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@profiles
+PREHOOK: query: create table school_summary(school string,cnt int) partitioned by (ds string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@school_summary
+POSTHOOK: query: create table school_summary(school string,cnt int) partitioned by (ds string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@school_summary
+PREHOOK: query: create table gender_summary(gender int, cnt int) partitioned by (ds string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@gender_summary
+POSTHOOK: query: create table gender_summary(gender int, cnt int) partitioned by (ds string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@gender_summary
+PREHOOK: query: insert into status_updates values (1, "status_1", "2009-03-20")
+PREHOOK: type: QUERY
+PREHOOK: Input: default@values__tmp__table__1
+PREHOOK: Output: default@status_updates
+POSTHOOK: query: insert into status_updates values (1, "status_1", "2009-03-20")
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@values__tmp__table__1
+POSTHOOK: Output: default@status_updates
+POSTHOOK: Lineage: status_updates.ds SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ]
+POSTHOOK: Lineage: status_updates.status SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+POSTHOOK: Lineage: status_updates.userid EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: insert into profiles values (1, "school_1", 0)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@values__tmp__table__2
+PREHOOK: Output: default@profiles
+POSTHOOK: query: insert into profiles values (1, "school_1", 0)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@values__tmp__table__2
+POSTHOOK: Output: default@profiles
+POSTHOOK: Lineage: profiles.gender EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col3, type:string, comment:), ]
+POSTHOOK: Lineage: profiles.school SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+POSTHOOK: Lineage: profiles.userid EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: FROM (SELECT a.status, b.school, b.gender
+FROM status_updates a JOIN profiles b
+ON (a.userid = b.userid and
+a.ds='2009-03-20' )
+) subq1
+INSERT OVERWRITE TABLE gender_summary
+PARTITION(ds='2009-03-20')
+SELECT subq1.gender, COUNT(1) GROUP BY subq1.gender
+INSERT OVERWRITE TABLE school_summary
+PARTITION(ds='2009-03-20')
+SELECT subq1.school, COUNT(1) GROUP BY subq1.school
+PREHOOK: type: QUERY
+PREHOOK: Input: default@profiles
+PREHOOK: Input: default@status_updates
+PREHOOK: Output: default@gender_summary@ds=2009-03-20
+PREHOOK: Output: default@school_summary@ds=2009-03-20
+POSTHOOK: query: FROM (SELECT a.status, b.school, b.gender
+FROM status_updates a JOIN profiles b
+ON (a.userid = b.userid and
+a.ds='2009-03-20' )
+) subq1
+INSERT OVERWRITE TABLE gender_summary
+PARTITION(ds='2009-03-20')
+SELECT subq1.gender, COUNT(1) GROUP BY subq1.gender
+INSERT OVERWRITE TABLE school_summary
+PARTITION(ds='2009-03-20')
+SELECT subq1.school, COUNT(1) GROUP BY subq1.school
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@profiles
+POSTHOOK: Input: default@status_updates
+POSTHOOK: Output: default@gender_summary@ds=2009-03-20
+POSTHOOK: Output: default@school_summary@ds=2009-03-20
+POSTHOOK: Lineage: gender_summary PARTITION(ds=2009-03-20).cnt EXPRESSION [(status_updates)a.null, (profiles)b.null, ]
+POSTHOOK: Lineage: gender_summary PARTITION(ds=2009-03-20).gender SIMPLE [(profiles)b.FieldSchema(name:gender, type:int, comment:null), ]
+POSTHOOK: Lineage: school_summary PARTITION(ds=2009-03-20).cnt EXPRESSION [(status_updates)a.null, (profiles)b.null, ]
+POSTHOOK: Lineage: school_summary PARTITION(ds=2009-03-20).school SIMPLE [(profiles)b.FieldSchema(name:school, type:string, comment:null), ]
+PREHOOK: query: select * from school_summary
+PREHOOK: type: QUERY
+PREHOOK: Input: default@school_summary
+PREHOOK: Input: default@school_summary@ds=2009-03-20
+#### A masked pattern was here ####
+POSTHOOK: query: select * from school_summary
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@school_summary
+POSTHOOK: Input: default@school_summary@ds=2009-03-20
+#### A masked pattern was here ####
+school_1 1 2009-03-20
+PREHOOK: query: select * from gender_summary
+PREHOOK: type: QUERY
+PREHOOK: Input: default@gender_summary
+PREHOOK: Input: default@gender_summary@ds=2009-03-20
+#### A masked pattern was here ####
+POSTHOOK: query: select * from gender_summary
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@gender_summary
+POSTHOOK: Input: default@gender_summary@ds=2009-03-20
+#### A masked pattern was here ####
+0 1 2009-03-20
[40/50] [abbrv] hive git commit: HIVE-12937 : DbNotificationListener
unable to clean up old notification events (Sushanth Sowmyan,
reviewed by Sergey Shelukhin)
Posted by jd...@apache.org.
HIVE-12937 : DbNotificationListener unable to clean up old notification events (Sushanth Sowmyan, reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1de97bc5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1de97bc5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1de97bc5
Branch: refs/heads/llap
Commit: 1de97bc5fad323ae3bd48ebb39e6e68a3581e099
Parents: 8c8ff3f
Author: Sushanth Sowmyan <kh...@gmail.com>
Authored: Tue Mar 29 11:21:23 2016 -0700
Committer: Sushanth Sowmyan <kh...@gmail.com>
Committed: Tue Mar 29 11:24:36 2016 -0700
----------------------------------------------------------------------
.../listener/TestDbNotificationListener.java | 18 ++++++++++++++++++
.../apache/hadoop/hive/metastore/ObjectStore.java | 2 +-
2 files changed, 19 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/1de97bc5/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
----------------------------------------------------------------------
diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
index 6caf3fe..1360563 100644
--- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
+++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
@@ -57,6 +57,7 @@ import java.util.Map;
public class TestDbNotificationListener {
private static final Logger LOG = LoggerFactory.getLogger(TestDbNotificationListener.class.getName());
+ private static final int EVENTS_TTL = 30;
private static Map<String, String> emptyParameters = new HashMap<String, String>();
private static IMetaStoreClient msClient;
private static Driver driver;
@@ -68,6 +69,7 @@ public class TestDbNotificationListener {
HiveConf conf = new HiveConf();
conf.setVar(HiveConf.ConfVars.METASTORE_EVENT_LISTENERS,
DbNotificationListener.class.getName());
+ conf.setVar(HiveConf.ConfVars.METASTORE_EVENT_DB_LISTENER_TTL, String.valueOf(EVENTS_TTL)+"s");
conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
conf.setBoolVar(HiveConf.ConfVars.FIRE_EVENTS_FOR_DML, true);
conf.setVar(HiveConf.ConfVars.DYNAMICPARTITIONINGMODE, "nonstrict");
@@ -565,4 +567,20 @@ public class TestDbNotificationListener {
assertEquals(firstEventId + 19, event.getEventId());
assertEquals(HCatConstants.HCAT_DROP_PARTITION_EVENT, event.getEventType());
}
+
+ @Test
+ public void cleanupNotifs() throws Exception {
+ Database db = new Database("cleanup1","no description","file:/tmp", emptyParameters);
+ msClient.createDatabase(db);
+ msClient.dropDatabase("cleanup1");
+
+ NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null);
+ assertEquals(2, rsp.getEventsSize());
+
+ // sleep for expiry time, and then fetch again
+ Thread.sleep(EVENTS_TTL * 2 * 1000); // sleep twice the TTL interval - things should have been cleaned by then.
+
+ NotificationEventResponse rsp2 = msClient.getNextNotification(firstEventId, 0, null);
+ assertEquals(0, rsp2.getEventsSize());
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/1de97bc5/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index 35adb39..ac293b9 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -7827,7 +7827,7 @@ public class ObjectStore implements RawStore, Configurable {
query.declareParameters("java.lang.Integer tooOld");
Collection<MNotificationLog> toBeRemoved = (Collection) query.execute(tooOld);
if (toBeRemoved != null && toBeRemoved.size() > 0) {
- pm.deletePersistent(toBeRemoved);
+ pm.deletePersistentAll(toBeRemoved);
}
commited = commitTransaction();
} finally {
[21/50] [abbrv] hive git commit: HIVE-13319 : Fix passing external
handles in task display (Rajat Khandelwal, reviewed by amareshwari)
Posted by jd...@apache.org.
HIVE-13319 : Fix passing external handles in task display (Rajat Khandelwal, reviewed by amareshwari)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/41a30b59
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/41a30b59
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/41a30b59
Branch: refs/heads/llap
Commit: 41a30b59de35601211657b65a20b9d418958fb58
Parents: 69cfd35
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Mon Mar 28 09:45:43 2016 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Mon Mar 28 09:45:43 2016 +0530
----------------------------------------------------------------------
ql/src/java/org/apache/hadoop/hive/ql/QueryDisplay.java | 11 +++++++----
.../test/org/apache/hive/service/cli/CLIServiceTest.java | 8 ++++++++
2 files changed, 15 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/41a30b59/ql/src/java/org/apache/hadoop/hive/ql/QueryDisplay.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/QueryDisplay.java b/ql/src/java/org/apache/hadoop/hive/ql/QueryDisplay.java
index 467dab6..d582bc0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/QueryDisplay.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/QueryDisplay.java
@@ -72,7 +72,7 @@ public class QueryDisplay {
private Long endTime;
private String taskId;
- private String taskExternalHandle;
+ private String externalHandle;
public Task.TaskState taskState;
private StageType taskType;
@@ -85,7 +85,7 @@ public class QueryDisplay {
}
public TaskDisplay(Task task) {
taskId = task.getId();
- taskExternalHandle = task.getExternalHandle();
+ externalHandle = task.getExternalHandle();
taskType = task.getType();
name = task.getName();
requireLock = task.requireLock();
@@ -150,12 +150,15 @@ public class QueryDisplay {
}
public synchronized String getExternalHandle() {
- return taskExternalHandle;
+ return externalHandle;
}
public synchronized <T extends Serializable> void updateStatus(Task<T> tTask) {
this.taskState = tTask.getTaskState();
- switch(taskState) {
+ if (externalHandle == null && tTask.getExternalHandle() != null) {
+ this.externalHandle = tTask.getExternalHandle();
+ }
+ switch (taskState) {
case RUNNING:
beginTime = System.currentTimeMillis();
break;
http://git-wip-us.apache.org/repos/asf/hive/blob/41a30b59/service/src/test/org/apache/hive/service/cli/CLIServiceTest.java
----------------------------------------------------------------------
diff --git a/service/src/test/org/apache/hive/service/cli/CLIServiceTest.java b/service/src/test/org/apache/hive/service/cli/CLIServiceTest.java
index e145eb4..698b13d 100644
--- a/service/src/test/org/apache/hive/service/cli/CLIServiceTest.java
+++ b/service/src/test/org/apache/hive/service/cli/CLIServiceTest.java
@@ -45,6 +45,7 @@ import org.codehaus.jackson.type.TypeReference;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.plan.api.StageType;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.junit.After;
@@ -676,6 +677,7 @@ public abstract class CLIServiceTest {
switch (taskDisplay.taskState) {
case INITIALIZED:
case QUEUED:
+ assertNull(taskDisplay.getExternalHandle());
assertNull(taskDisplay.getBeginTime());
assertNull(taskDisplay.getEndTime());
assertNull(taskDisplay.getElapsedTime());
@@ -683,6 +685,9 @@ public abstract class CLIServiceTest {
assertNull(taskDisplay.getReturnValue());
break;
case RUNNING:
+ if (taskDisplay.getTaskType() == StageType.MAPRED || taskDisplay.getTaskType() == StageType.MAPREDLOCAL) {
+ assertNotNull(taskDisplay.getExternalHandle());
+ }
assertNotNull(taskDisplay.getBeginTime());
assertNull(taskDisplay.getEndTime());
assertNotNull(taskDisplay.getElapsedTime());
@@ -690,6 +695,9 @@ public abstract class CLIServiceTest {
assertNull(taskDisplay.getReturnValue());
break;
case FINISHED:
+ if (taskDisplay.getTaskType() == StageType.MAPRED || taskDisplay.getTaskType() == StageType.MAPREDLOCAL) {
+ assertNotNull(taskDisplay.getExternalHandle());
+ }
assertNotNull(taskDisplay.getBeginTime());
assertNotNull(taskDisplay.getEndTime());
assertNotNull(taskDisplay.getElapsedTime());
[28/50] [abbrv] hive git commit: HIVE-13111: Fix timestamp /
interval_day_time wrong results with HIVE-9862 (Matt McCline,
reviewed by Jason Dere)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTypeCasts.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTypeCasts.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTypeCasts.java
index 038e382..1e41fce 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTypeCasts.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTypeCasts.java
@@ -23,13 +23,19 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import java.io.UnsupportedEncodingException;
+import java.math.BigDecimal;
+import java.math.MathContext;
+import java.math.RoundingMode;
+import java.sql.Timestamp;
import java.util.Arrays;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
import junit.framework.Assert;
import org.apache.hadoop.hive.common.type.Decimal128;
import org.apache.hadoop.hive.common.type.HiveDecimal;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
+import org.apache.hadoop.hive.common.type.RandomTypeUtil;
import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
@@ -39,6 +45,7 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.*;
import org.apache.hadoop.hive.ql.exec.vector.expressions.*;
import org.apache.hadoop.hive.serde2.io.TimestampWritable;
+import org.apache.hadoop.hive.serde2.typeinfo.HiveDecimalUtils;
import org.junit.Test;
/**
@@ -84,8 +91,8 @@ public class TestVectorTypeCasts {
b.cols[0].noNulls = true;
VectorExpression expr = new CastDoubleToTimestamp(0, 1);
expr.evaluate(b);
- Assert.assertEquals(0.0, resultV.getTimestampSecondsWithFractionalNanos(3));
- Assert.assertEquals(0.5d, resultV.getTimestampSecondsWithFractionalNanos(4));
+ Assert.assertEquals(0.0, TimestampWritable.getDouble(resultV.asScratchTimestamp(3)));
+ Assert.assertEquals(0.5d, TimestampWritable.getDouble(resultV.asScratchTimestamp(4)));
}
@Test
@@ -103,39 +110,51 @@ public class TestVectorTypeCasts {
@Test
public void testCastLongToTimestamp() {
- VectorizedRowBatch b = TestVectorMathFunctions.getVectorizedRowBatchLongInTimestampOut();
+ long[] longValues = new long[500];
+ VectorizedRowBatch b = TestVectorMathFunctions.getVectorizedRowBatchLongInTimestampOut(longValues);
TimestampColumnVector resultV = (TimestampColumnVector) b.cols[1];
b.cols[0].noNulls = true;
VectorExpression expr = new CastLongToTimestamp(0, 1);
expr.evaluate(b);
- Assert.assertEquals(-2, resultV.getTimestampSeconds(0));
- Assert.assertEquals(2, resultV.getTimestampSeconds(1));
+ for (int i = 0; i < longValues.length; i++) {
+ Timestamp timestamp = resultV.asScratchTimestamp(i);
+ long actual = TimestampWritable.getLong(timestamp);
+ assertEquals(actual, longValues[i]);
+ }
}
@Test
public void testCastTimestampToLong() {
- VectorizedRowBatch b = TestVectorMathFunctions.getVectorizedRowBatchTimestampInLongOut();
+ long[] longValues = new long[500];
+ VectorizedRowBatch b = TestVectorMathFunctions.getVectorizedRowBatchTimestampInLongOut(longValues);
TimestampColumnVector inV = (TimestampColumnVector) b.cols[0];
- inV.set(0, new PisaTimestamp(0, PisaTimestamp.NANOSECONDS_PER_SECOND)); // Make one entry produce interesting result
- // (1 sec after epoch).
-
LongColumnVector resultV = (LongColumnVector) b.cols[1];
b.cols[0].noNulls = true;
VectorExpression expr = new CastTimestampToLong(0, 1);
expr.evaluate(b);
- Assert.assertEquals(1, resultV.vector[0]);
+ for (int i = 0; i < longValues.length; i++) {
+ long actual = resultV.vector[i];
+ long timestampLong = inV.getTimestampAsLong(i);
+ if (actual != timestampLong) {
+ assertTrue(false);
+ }
+ }
}
@Test
public void testCastTimestampToDouble() {
- VectorizedRowBatch b = TestVectorMathFunctions.getVectorizedRowBatchTimestampInDoubleOut();
+ double[] doubleValues = new double[500];
+ VectorizedRowBatch b = TestVectorMathFunctions.getVectorizedRowBatchTimestampInDoubleOut(doubleValues);
TimestampColumnVector inV = (TimestampColumnVector) b.cols[0];
DoubleColumnVector resultV = (DoubleColumnVector) b.cols[1];
b.cols[0].noNulls = true;
VectorExpression expr = new CastTimestampToDouble(0, 1);
expr.evaluate(b);
- Assert.assertEquals(-1E-9D , resultV.vector[1]);
- Assert.assertEquals(1E-9D, resultV.vector[3]);
+ for (int i = 0; i < doubleValues.length; i++) {
+ double actual = resultV.vector[i];
+ double doubleValue = TimestampWritable.getDouble(inV.asScratchTimestamp(i));
+ assertEquals(actual, doubleValue, 0.000000001F);
+ }
}
public byte[] toBytes(String s) {
@@ -356,16 +375,19 @@ public class TestVectorTypeCasts {
@Test
public void testCastDecimalToTimestamp() {
- VectorizedRowBatch b = getBatchDecimalTimestamp();
+ double[] doubleValues = new double[500];
+ VectorizedRowBatch b = getBatchDecimalTimestamp(doubleValues);
VectorExpression expr = new CastDecimalToTimestamp(0, 1);
expr.evaluate(b);
TimestampColumnVector r = (TimestampColumnVector) b.cols[1];
- assertEquals(1111111111L, r.getNanoOfDay(0));
- assertEquals(0L, r.getEpochDay(0));
- assertEquals(-2222222222L, r.getNanoOfDay(1));
- assertEquals(0L, r.getEpochDay(1));
- assertEquals(999999999L, r.getNanoOfDay(2));
- assertEquals(365L, r.getEpochDay(2));
+ for (int i = 0; i < doubleValues.length; i++) {
+ Timestamp timestamp = r.asScratchTimestamp(i);
+ double asDouble = TimestampWritable.getDouble(timestamp);
+ double expectedDouble = doubleValues[i];
+ if (expectedDouble != asDouble) {
+ assertTrue(false);
+ }
+ }
}
private VectorizedRowBatch getBatchDecimalLong2() {
@@ -384,19 +406,25 @@ public class TestVectorTypeCasts {
return b;
}
- private VectorizedRowBatch getBatchDecimalTimestamp() {
+ private VectorizedRowBatch getBatchDecimalTimestamp(double[] doubleValues) {
VectorizedRowBatch b = new VectorizedRowBatch(2);
DecimalColumnVector dv;
- short scale = 9;
- b.cols[0] = dv = new DecimalColumnVector(18, scale);
- b.cols[1] = new TimestampColumnVector();
-
- b.size = 3;
-
- dv.vector[0].set(HiveDecimal.create("1.111111111").setScale(scale));
- dv.vector[1].set(HiveDecimal.create("-2.222222222").setScale(scale));
- dv.vector[2].set(HiveDecimal.create("31536000.999999999").setScale(scale));
-
+ b.cols[0] = dv = new DecimalColumnVector(doubleValues.length, HiveDecimal.SYSTEM_DEFAULT_PRECISION, HiveDecimal.SYSTEM_DEFAULT_SCALE);
+ b.cols[1] = new TimestampColumnVector(doubleValues.length);
+ dv.noNulls = true;
+ Random r = new Random(94830);
+ for (int i = 0; i < doubleValues.length; i++) {
+ long millis = RandomTypeUtil.randomMillis(r);
+ Timestamp ts = new Timestamp(millis);
+ int nanos = RandomTypeUtil.randomNanos(r);
+ ts.setNanos(nanos);
+ TimestampWritable tsw = new TimestampWritable(ts);
+ double asDouble = tsw.getDouble();
+ doubleValues[i] = asDouble;
+ HiveDecimal hiveDecimal = HiveDecimal.create(new BigDecimal(asDouble));
+ dv.set(i, hiveDecimal);
+ }
+ b.size = doubleValues.length;
return b;
}
@@ -422,14 +450,44 @@ public class TestVectorTypeCasts {
return b;
}
- private VectorizedRowBatch getBatchTimestampDecimal() {
+
+ public static final long NANOSECONDS_PER_SECOND = TimeUnit.SECONDS.toNanos(1);
+ public static final long MILLISECONDS_PER_SECOND = TimeUnit.SECONDS.toMillis(1);
+ public static final long NANOSECONDS_PER_MILLISSECOND = TimeUnit.MILLISECONDS.toNanos(1);
+
+ private VectorizedRowBatch getBatchTimestampDecimal(HiveDecimal[] hiveDecimalValues) {
+ Random r = new Random(994);
VectorizedRowBatch b = new VectorizedRowBatch(2);
TimestampColumnVector tcv;
- b.cols[0] = tcv = new TimestampColumnVector();
- b.cols[1] = new DecimalColumnVector(18, 2);
- tcv.set(0, new PisaTimestamp( 0, 0));
- tcv.set(1, new PisaTimestamp( 0, -1));
- tcv.set(2, new PisaTimestamp( 99999999999999L / PisaTimestamp.NANOSECONDS_PER_DAY, 99999999999999L % PisaTimestamp.NANOSECONDS_PER_DAY));
+ b.cols[0] = tcv = new TimestampColumnVector(hiveDecimalValues.length);
+ b.cols[1] = new DecimalColumnVector(hiveDecimalValues.length, HiveDecimal.SYSTEM_DEFAULT_PRECISION, HiveDecimal.SYSTEM_DEFAULT_SCALE);
+ for (int i = 0; i < hiveDecimalValues.length; i++) {
+ int optionalNanos = 0;
+ switch (r.nextInt(4)) {
+ case 0:
+ // No nanos.
+ break;
+ case 1:
+ optionalNanos = r.nextInt((int) NANOSECONDS_PER_SECOND);
+ break;
+ case 2:
+ // Limit to milliseconds only...
+ optionalNanos = r.nextInt((int) MILLISECONDS_PER_SECOND) * (int) NANOSECONDS_PER_MILLISSECOND;
+ break;
+ case 3:
+ // Limit to below milliseconds only...
+ optionalNanos = r.nextInt((int) NANOSECONDS_PER_MILLISSECOND);
+ break;
+ }
+ long millis = RandomTypeUtil.randomMillis(r);
+ Timestamp ts = new Timestamp(millis);
+ ts.setNanos(optionalNanos);
+ TimestampWritable tsw = new TimestampWritable(ts);
+ hiveDecimalValues[i] = tsw.getHiveDecimal();
+
+ tcv.set(i, ts);
+ }
+ b.size = hiveDecimalValues.length;
return b;
}
@@ -440,9 +498,18 @@ public class TestVectorTypeCasts {
expr.evaluate(b);
DecimalColumnVector r = (DecimalColumnVector) b.cols[1];
- assertTrue(r.vector[0].getHiveDecimal().equals(HiveDecimal.create("0.0")));
- assertTrue(r.vector[1].getHiveDecimal().equals(HiveDecimal.create("-1.0")));
- assertTrue(r.vector[2].getHiveDecimal().equals(HiveDecimal.create("99999999999999")));
+ HiveDecimal hd0 = HiveDecimal.create("0.0");
+ if (!hd0.equals(r.vector[0].getHiveDecimal())) {
+ assertTrue(false);
+ }
+ HiveDecimal hd1 = HiveDecimal.create("-1.0");
+ if (!hd1.equals(r.vector[1].getHiveDecimal())) {
+ assertTrue(false);
+ }
+ HiveDecimal hd2 = HiveDecimal.create("99999999999999");
+ if (!hd2.equals(r.vector[2].getHiveDecimal())) {
+ assertTrue(false);
+ }
}
private VectorizedRowBatch getBatchDoubleDecimal() {
@@ -496,25 +563,37 @@ public class TestVectorTypeCasts {
// The input timestamps are stored as long values
// measured in nanoseconds from the epoch.
- VectorizedRowBatch b = getBatchTimestampDecimal();
+ HiveDecimal[] hiveDecimalValues = new HiveDecimal[500];
+ VectorizedRowBatch b = getBatchTimestampDecimal(hiveDecimalValues);
VectorExpression expr = new CastTimestampToDecimal(0, 1);
TimestampColumnVector inT = (TimestampColumnVector) b.cols[0];
- inT.set(1, new PisaTimestamp(0, -1990000000L));
expr.evaluate(b);
DecimalColumnVector r = (DecimalColumnVector) b.cols[1];
- assertTrue(r.vector[0].getHiveDecimal().equals(HiveDecimal.create("0.00")));
- assertTrue(r.vector[1].getHiveDecimal().equals(HiveDecimal.create("-1.99")));
- assertTrue(r.vector[2].getHiveDecimal().equals(HiveDecimal.create("100000.00")));
+ for (int i = 0; i < hiveDecimalValues.length; i++) {
+ HiveDecimal hiveDecimal = r.vector[i].getHiveDecimal();
+ HiveDecimal expectedHiveDecimal = hiveDecimalValues[i];
+ if (!hiveDecimal.equals(expectedHiveDecimal)) {
+ assertTrue(false);
+ }
+ }
// Try again with a value that won't fit in 5 digits, to make
// sure that NULL is produced.
- b = getBatchTimestampDecimalPrec5Scale2();
+ b.cols[1] = r = new DecimalColumnVector(hiveDecimalValues.length, 5, 2);
expr.evaluate(b);
r = (DecimalColumnVector) b.cols[1];
- assertFalse(r.noNulls);
- assertFalse(r.isNull[0]);
- assertFalse(r.isNull[1]);
- assertTrue(r.isNull[2]);
+ for (int i = 0; i < hiveDecimalValues.length; i++) {
+ HiveDecimal hiveDecimal = r.vector[i].getHiveDecimal();
+ HiveDecimal expectedHiveDecimal = hiveDecimalValues[i];
+ if (HiveDecimal.enforcePrecisionScale(expectedHiveDecimal, 5, 2) == null) {
+ assertTrue(r.isNull[i]);
+ } else {
+ assertTrue(!r.isNull[i]);
+ if (!hiveDecimal.equals(expectedHiveDecimal)) {
+ assertTrue(false);
+ }
+ }
+ }
}
/* This batch has output decimal column precision 5 and scale 2.
@@ -533,41 +612,6 @@ public class TestVectorTypeCasts {
return b;
}
- private VectorizedRowBatch getBatchTimestampDecimalPrec5Scale2() {
- VectorizedRowBatch b = new VectorizedRowBatch(2);
- TimestampColumnVector tcv;
- b.cols[0] = tcv = new TimestampColumnVector();
- b.cols[1] = new DecimalColumnVector(5, 2);
- tcv.set(0, new PisaTimestamp(0, 0));
- tcv.set(1, new PisaTimestamp(0, -1));
- tcv.set(2, new PisaTimestamp(99999999999999L / PisaTimestamp.NANOSECONDS_PER_DAY, 99999999999999L % PisaTimestamp.NANOSECONDS_PER_DAY));
- return b;
- }
-
- /*
- @Test
- public void testCastDecimalToDecimal() {
-
- // test casting from one precision and scale to another.
- VectorizedRowBatch b = getBatchDecimalDecimal();
- VectorExpression expr = new CastDecimalToDecimal(0, 1);
- expr.evaluate(b);
- DecimalColumnVector r = (DecimalColumnVector) b.cols[1];
- assertTrue(r.vector[0].getHiveDecimal().equals(HiveDecimal.create("10.00", (short) 2)));
- assertFalse(r.noNulls);
- assertTrue(r.isNull[1]);
-
- // test an increase in precision/scale
- b = getBatchDecimalDecimal();
- expr = new CastDecimalToDecimal(1, 0);
- expr.evaluate(b);
- r = (DecimalColumnVector) b.cols[0];
- assertTrue(r.vector[0].getHiveDecimal().equals(HiveDecimal.create("100.01", (short) 4)));
- assertTrue(r.vector[1].getHiveDecimal().equals(HiveDecimal.create("-200.02", (short) 4)));
- assertTrue(r.noNulls);
- }
- */
-
private VectorizedRowBatch getBatchDecimalDecimal() {
VectorizedRowBatch b = new VectorizedRowBatch(2);
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeVectorRowBatchFromObjectIterables.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeVectorRowBatchFromObjectIterables.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeVectorRowBatchFromObjectIterables.java
index ab86082..98849c3 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeVectorRowBatchFromObjectIterables.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeVectorRowBatchFromObjectIterables.java
@@ -27,7 +27,6 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.hadoop.hive.common.type.HiveDecimal;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
@@ -111,7 +110,7 @@ public class FakeVectorRowBatchFromObjectIterables extends FakeVectorRowBatchBas
Object value) {
TimestampColumnVector lcv = (TimestampColumnVector) columnVector;
Timestamp t = (Timestamp) value;
- lcv.set(row, new PisaTimestamp(t));
+ lcv.set(row, t);
}
};
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/VectorizedRowGroupGenUtil.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/VectorizedRowGroupGenUtil.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/VectorizedRowGroupGenUtil.java
index 649e52b..84717b1 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/VectorizedRowGroupGenUtil.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/VectorizedRowGroupGenUtil.java
@@ -22,7 +22,6 @@ import java.sql.Timestamp;
import java.util.Random;
import org.apache.hadoop.hive.common.type.HiveDecimal;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
import org.apache.hadoop.hive.common.type.RandomTypeUtil;
import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
@@ -81,7 +80,7 @@ public class VectorizedRowGroupGenUtil {
}
public static TimestampColumnVector generateTimestampColumnVector(
- boolean nulls, boolean repeating, int size, Random rand) {
+ boolean nulls, boolean repeating, int size, Random rand, Timestamp[] timestampValues) {
TimestampColumnVector tcv = new TimestampColumnVector(size);
tcv.noNulls = !nulls;
@@ -95,10 +94,17 @@ public class VectorizedRowGroupGenUtil {
if(nulls && (repeating || i % nullFrequency == 0)) {
tcv.isNull[i] = true;
tcv.setNullValue(i);
-
+ timestampValues[i] = null;
}else {
tcv.isNull[i] = false;
- tcv.set(i, repeating ? repeatingTimestamp : RandomTypeUtil.getRandTimestamp(rand));
+ if (!repeating) {
+ Timestamp randomTimestamp = RandomTypeUtil.getRandTimestamp(rand);
+ tcv.set(i, randomTimestamp);
+ timestampValues[i] = randomTimestamp;
+ } else {
+ tcv.set(i, repeatingTimestamp);
+ timestampValues[i] = repeatingTimestamp;
+ }
}
}
return tcv;
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
index c88f6d8..85923a8 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
@@ -1946,7 +1946,7 @@ public class TestInputOutputFormat {
long millis = (long) i * MILLIS_IN_DAY;
millis -= LOCAL_TIMEZONE.getOffset(millis);
assertEquals("checking timestamp " + i, millis,
- timestampColumn.getTimestampMilliseconds(i));
+ timestampColumn.getTime(i));
}
assertEquals(false, reader.next(key, value));
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java
index 3843c6d..1a97a6d 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java
@@ -541,7 +541,10 @@ public class TestOrcFile {
int idx = 0;
while (rows.hasNext()) {
Object row = rows.next(null);
- assertEquals(tslist.get(idx++).getNanos(), ((TimestampWritable) row).getNanos());
+ Timestamp tlistTimestamp = tslist.get(idx++);
+ if (tlistTimestamp.getNanos() != ((TimestampWritable) row).getNanos()) {
+ assertTrue(false);
+ }
}
assertEquals(0, writer.getSchema().getMaximumId());
boolean[] expected = new boolean[] {false};
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorOrcFile.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorOrcFile.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorOrcFile.java
index 4ca20c5..a82d672 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorOrcFile.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorOrcFile.java
@@ -24,7 +24,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.type.HiveDecimal;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
@@ -527,7 +526,7 @@ public class TestVectorOrcFile {
batch.size = tslist.size();
for (int i=0; i < tslist.size(); ++i) {
Timestamp ts = tslist.get(i);
- vec.set(i, new PisaTimestamp(ts));
+ vec.set(i, ts);
}
writer.addRowBatch(batch);
writer.close();
@@ -1345,8 +1344,8 @@ public class TestVectorOrcFile {
for (int ms = 1000; ms < 2000; ++ms) {
TimestampColumnVector timestampColVector = (TimestampColumnVector) batch.cols[0];
timestampColVector.set(ms - 1000,
- new PisaTimestamp(Timestamp.valueOf(year +
- "-05-05 12:34:56." + ms)));
+ Timestamp.valueOf(year +
+ "-05-05 12:34:56." + ms));
((LongColumnVector) batch.cols[1]).vector[ms - 1000] =
new DateWritable(new Date(year - 1900, 11, 25)).getDays();
}
@@ -1385,7 +1384,7 @@ public class TestVectorOrcFile {
UnionColumnVector union = (UnionColumnVector) batch.cols[1];
if (ts != null) {
TimestampColumnVector timestampColVector = (TimestampColumnVector) batch.cols[0];
- timestampColVector.set(rowId, new PisaTimestamp(ts));
+ timestampColVector.set(rowId, ts);
} else {
batch.cols[0].isNull[rowId] = true;
batch.cols[0].noNulls = false;
@@ -2178,8 +2177,8 @@ public class TestVectorOrcFile {
((LongColumnVector) batch.cols[6]).vector[0] =
new DateWritable(new Date(111, 6, 1)).getDays();
((TimestampColumnVector) batch.cols[7]).set(0,
- new PisaTimestamp(new Timestamp(115, 9, 23, 10, 11, 59,
- 999999999)));
+ new Timestamp(115, 9, 23, 10, 11, 59,
+ 999999999));
((DecimalColumnVector) batch.cols[8]).vector[0] =
new HiveDecimalWritable("1.234567");
((BytesColumnVector) batch.cols[9]).setVal(0, "Echelon".getBytes());
@@ -2234,9 +2233,8 @@ public class TestVectorOrcFile {
new DateWritable(new Date(111, 6, 1)).getDays() + r;
Timestamp ts = new Timestamp(115, 9, 23, 10, 11, 59, 999999999);
- PisaTimestamp pisaTimestamp = new PisaTimestamp(ts);
- pisaTimestamp.addSeconds(pisaTimestamp, r, pisaTimestamp);
- ((TimestampColumnVector) batch.cols[7]).set(r, pisaTimestamp);
+ ts.setTime(ts.getTime() + r * 1000);
+ ((TimestampColumnVector) batch.cols[7]).set(r, ts);
((DecimalColumnVector) batch.cols[8]).vector[r] =
new HiveDecimalWritable("1.234567");
((BytesColumnVector) batch.cols[9]).setVal(r,
@@ -2378,8 +2376,10 @@ public class TestVectorOrcFile {
row.getFieldValue(5).toString());
assertEquals("row " + r, new Date(111, 6, 1 + r).toString(),
row.getFieldValue(6).toString());
+ Timestamp ts = new Timestamp(115, 9, 23, 10, 11, 59, 999999999);
+ ts.setTime(ts.getTime() + r * 1000);
assertEquals("row " + r,
- new Timestamp(115, 9, 23, 10, 11, 59 + r, 999999999).toString(),
+ ts.toString(),
row.getFieldValue(7).toString());
assertEquals("row " + r, "1.234567", row.getFieldValue(8).toString());
assertEquals("row " + r, Integer.toString(r),
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/test/queries/clientpositive/vector_interval_arithmetic.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vector_interval_arithmetic.q b/ql/src/test/queries/clientpositive/vector_interval_arithmetic.q
new file mode 100644
index 0000000..40c4c03
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/vector_interval_arithmetic.q
@@ -0,0 +1,174 @@
+set hive.cli.print.header=true;
+set hive.explain.user=false;
+set hive.fetch.task.conversion=none;
+
+create table unique_timestamps (tsval timestamp) STORED AS TEXTFILE;
+
+LOAD DATA LOCAL INPATH '../../data/files/timestamps.txt' OVERWRITE INTO TABLE unique_timestamps;
+
+create table interval_arithmetic_1 (dateval date, tsval timestamp) stored as orc;
+insert overwrite table interval_arithmetic_1
+ select cast(tsval as date), tsval from unique_timestamps;
+
+SET hive.vectorized.execution.enabled=true;
+
+-- interval year-month arithmetic
+explain
+select
+ dateval,
+ dateval - interval '2-2' year to month,
+ dateval - interval '-2-2' year to month,
+ dateval + interval '2-2' year to month,
+ dateval + interval '-2-2' year to month,
+ - interval '2-2' year to month + dateval,
+ interval '2-2' year to month + dateval
+from interval_arithmetic_1
+order by dateval;
+
+select
+ dateval,
+ dateval - interval '2-2' year to month,
+ dateval - interval '-2-2' year to month,
+ dateval + interval '2-2' year to month,
+ dateval + interval '-2-2' year to month,
+ - interval '2-2' year to month + dateval,
+ interval '2-2' year to month + dateval
+from interval_arithmetic_1
+order by dateval;
+
+explain
+select
+ dateval,
+ dateval - date '1999-06-07',
+ date '1999-06-07' - dateval,
+ dateval - dateval
+from interval_arithmetic_1
+order by dateval;
+
+select
+ dateval,
+ dateval - date '1999-06-07',
+ date '1999-06-07' - dateval,
+ dateval - dateval
+from interval_arithmetic_1
+order by dateval;
+
+explain
+select
+ tsval,
+ tsval - interval '2-2' year to month,
+ tsval - interval '-2-2' year to month,
+ tsval + interval '2-2' year to month,
+ tsval + interval '-2-2' year to month,
+ - interval '2-2' year to month + tsval,
+ interval '2-2' year to month + tsval
+from interval_arithmetic_1
+order by tsval;
+
+select
+ tsval,
+ tsval - interval '2-2' year to month,
+ tsval - interval '-2-2' year to month,
+ tsval + interval '2-2' year to month,
+ tsval + interval '-2-2' year to month,
+ - interval '2-2' year to month + tsval,
+ interval '2-2' year to month + tsval
+from interval_arithmetic_1
+order by tsval;
+
+explain
+select
+ interval '2-2' year to month + interval '3-3' year to month,
+ interval '2-2' year to month - interval '3-3' year to month
+from interval_arithmetic_1
+order by interval '2-2' year to month + interval '3-3' year to month
+limit 2;
+
+select
+ interval '2-2' year to month + interval '3-3' year to month,
+ interval '2-2' year to month - interval '3-3' year to month
+from interval_arithmetic_1
+order by interval '2-2' year to month + interval '3-3' year to month
+limit 2;
+
+
+-- interval day-time arithmetic
+explain
+select
+ dateval,
+ dateval - interval '99 11:22:33.123456789' day to second,
+ dateval - interval '-99 11:22:33.123456789' day to second,
+ dateval + interval '99 11:22:33.123456789' day to second,
+ dateval + interval '-99 11:22:33.123456789' day to second,
+ -interval '99 11:22:33.123456789' day to second + dateval,
+ interval '99 11:22:33.123456789' day to second + dateval
+from interval_arithmetic_1
+order by dateval;
+
+select
+ dateval,
+ dateval - interval '99 11:22:33.123456789' day to second,
+ dateval - interval '-99 11:22:33.123456789' day to second,
+ dateval + interval '99 11:22:33.123456789' day to second,
+ dateval + interval '-99 11:22:33.123456789' day to second,
+ -interval '99 11:22:33.123456789' day to second + dateval,
+ interval '99 11:22:33.123456789' day to second + dateval
+from interval_arithmetic_1
+order by dateval;
+
+explain
+select
+ dateval,
+ tsval,
+ dateval - tsval,
+ tsval - dateval,
+ tsval - tsval
+from interval_arithmetic_1
+order by dateval;
+
+select
+ dateval,
+ tsval,
+ dateval - tsval,
+ tsval - dateval,
+ tsval - tsval
+from interval_arithmetic_1
+order by dateval;
+
+explain
+select
+ tsval,
+ tsval - interval '99 11:22:33.123456789' day to second,
+ tsval - interval '-99 11:22:33.123456789' day to second,
+ tsval + interval '99 11:22:33.123456789' day to second,
+ tsval + interval '-99 11:22:33.123456789' day to second,
+ -interval '99 11:22:33.123456789' day to second + tsval,
+ interval '99 11:22:33.123456789' day to second + tsval
+from interval_arithmetic_1
+order by tsval;
+
+select
+ tsval,
+ tsval - interval '99 11:22:33.123456789' day to second,
+ tsval - interval '-99 11:22:33.123456789' day to second,
+ tsval + interval '99 11:22:33.123456789' day to second,
+ tsval + interval '-99 11:22:33.123456789' day to second,
+ -interval '99 11:22:33.123456789' day to second + tsval,
+ interval '99 11:22:33.123456789' day to second + tsval
+from interval_arithmetic_1
+order by tsval;
+
+explain
+select
+ interval '99 11:22:33.123456789' day to second + interval '10 9:8:7.123456789' day to second,
+ interval '99 11:22:33.123456789' day to second - interval '10 9:8:7.123456789' day to second
+from interval_arithmetic_1
+limit 2;
+
+select
+ interval '99 11:22:33.123456789' day to second + interval '10 9:8:7.123456789' day to second,
+ interval '99 11:22:33.123456789' day to second - interval '10 9:8:7.123456789' day to second
+from interval_arithmetic_1
+limit 2;
+
+drop table interval_arithmetic_1;
[22/50] [abbrv] hive git commit: HIVE-13358: Stats state is not
captured correctly: turn off stats optimizer for sampled table (Pengcheng
Xiong, reviewed by Ashutosh Chauhan)
Posted by jd...@apache.org.
HIVE-13358: Stats state is not captured correctly: turn off stats optimizer for sampled table (Pengcheng Xiong, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/77474581
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/77474581
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/77474581
Branch: refs/heads/llap
Commit: 77474581df4016e3899a986e079513087a945674
Parents: 41a30b5
Author: Pengcheng Xiong <px...@apache.org>
Authored: Sun Mar 27 22:30:29 2016 -0700
Committer: Pengcheng Xiong <px...@apache.org>
Committed: Sun Mar 27 22:30:42 2016 -0700
----------------------------------------------------------------------
.../hive/ql/optimizer/StatsOptimizer.java | 14 +-
.../sample_islocalmode_hook_use_metadata.q | 48 ++++
.../sample_islocalmode_hook_use_metadata.q.out | 230 +++++++++++++++++++
3 files changed, 289 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/77474581/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java
index 4091c0d..bc17fec 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java
@@ -100,9 +100,12 @@ public class StatsOptimizer extends Transform {
@Override
public ParseContext transform(ParseContext pctx) throws SemanticException {
- if (pctx.getFetchTask() != null || !pctx.getQueryProperties().isQuery() ||
- pctx.getQueryProperties().isAnalyzeRewrite() || pctx.getQueryProperties().isCTAS() ||
- pctx.getLoadFileWork().size() > 1 || !pctx.getLoadTableWork().isEmpty()) {
+ if (pctx.getFetchTask() != null || !pctx.getQueryProperties().isQuery()
+ || pctx.getQueryProperties().isAnalyzeRewrite() || pctx.getQueryProperties().isCTAS()
+ || pctx.getLoadFileWork().size() > 1 || !pctx.getLoadTableWork().isEmpty()
+ // If getNameToSplitSample is not empty, at least one of the source
+ // tables is being sampled and we can not optimize.
+ || !pctx.getNameToSplitSample().isEmpty()) {
return pctx;
}
@@ -251,6 +254,11 @@ public class StatsOptimizer extends Transform {
// looks like a subq plan.
return null;
}
+ if (tsOp.getConf().getRowLimit() != -1) {
+ // table is sampled. In some situation, we really can leverage row
+ // limit. In order to be safe, we do not use it now.
+ return null;
+ }
SelectOperator pselOp = (SelectOperator)stack.get(1);
for(ExprNodeDesc desc : pselOp.getConf().getColList()) {
if (!((desc instanceof ExprNodeColumnDesc) || (desc instanceof ExprNodeConstantDesc))) {
http://git-wip-us.apache.org/repos/asf/hive/blob/77474581/ql/src/test/queries/clientpositive/sample_islocalmode_hook_use_metadata.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/sample_islocalmode_hook_use_metadata.q b/ql/src/test/queries/clientpositive/sample_islocalmode_hook_use_metadata.q
new file mode 100644
index 0000000..ac915b5
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/sample_islocalmode_hook_use_metadata.q
@@ -0,0 +1,48 @@
+set hive.mapred.mode=nonstrict;
+set hive.exec.submitviachild=true;
+set hive.exec.submit.local.task.via.child=true;
+set hive.input.format=org.apache.hadoop.hive.ql.io.CombineHiveInputFormat;
+set mapred.max.split.size=300;
+set mapred.min.split.size=300;
+set mapred.min.split.size.per.node=300;
+set mapred.min.split.size.per.rack=300;
+set hive.exec.mode.local.auto=true;
+set hive.merge.smallfiles.avgsize=1;
+set hive.compute.query.using.stats=true;
+
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS( 0.20S)
+
+-- create file inputs
+create table sih_i_part (key int, value string) partitioned by (p string);
+insert overwrite table sih_i_part partition (p='1') select key, value from src;
+insert overwrite table sih_i_part partition (p='2') select key+10000, value from src;
+insert overwrite table sih_i_part partition (p='3') select key+20000, value from src;
+create table sih_src as select key, value from sih_i_part order by key, value;
+create table sih_src2 as select key, value from sih_src order by key, value;
+
+set hive.exec.post.hooks = org.apache.hadoop.hive.ql.hooks.VerifyIsLocalModeHook;
+set mapreduce.framework.name=yarn;
+set mapreduce.jobtracker.address=localhost:58;
+set hive.sample.seednumber=7;
+
+-- Relaxing hive.exec.mode.local.auto.input.files.max=1.
+-- Hadoop20 will not generate more splits than there are files (one).
+-- Hadoop23 generate splits correctly (four), hence the max needs to be adjusted to ensure running in local mode.
+-- Default value is hive.exec.mode.local.auto.input.files.max=4 which produces expected behavior on Hadoop23.
+-- hive.sample.seednumber is required because Hadoop23 generates multiple splits and tablesample is non-repeatable without it.
+
+-- sample split, running locally limited by num tasks
+
+desc formatted sih_src;
+
+explain select count(1) from sih_src;
+
+select count(1) from sih_src;
+
+explain select count(1) from sih_src tablesample(1 percent);
+
+select count(1) from sih_src tablesample(1 percent);
+
+explain select count(1) from sih_src tablesample(10 rows);
+
+select count(1) from sih_src tablesample(10 rows);
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/77474581/ql/src/test/results/clientpositive/sample_islocalmode_hook_use_metadata.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/sample_islocalmode_hook_use_metadata.q.out b/ql/src/test/results/clientpositive/sample_islocalmode_hook_use_metadata.q.out
new file mode 100644
index 0000000..d268837
--- /dev/null
+++ b/ql/src/test/results/clientpositive/sample_islocalmode_hook_use_metadata.q.out
@@ -0,0 +1,230 @@
+PREHOOK: query: -- EXCLUDE_HADOOP_MAJOR_VERSIONS( 0.20S)
+
+-- create file inputs
+create table sih_i_part (key int, value string) partitioned by (p string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@sih_i_part
+POSTHOOK: query: -- EXCLUDE_HADOOP_MAJOR_VERSIONS( 0.20S)
+
+-- create file inputs
+create table sih_i_part (key int, value string) partitioned by (p string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@sih_i_part
+PREHOOK: query: insert overwrite table sih_i_part partition (p='1') select key, value from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@sih_i_part@p=1
+POSTHOOK: query: insert overwrite table sih_i_part partition (p='1') select key, value from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@sih_i_part@p=1
+POSTHOOK: Lineage: sih_i_part PARTITION(p=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: sih_i_part PARTITION(p=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: insert overwrite table sih_i_part partition (p='2') select key+10000, value from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@sih_i_part@p=2
+POSTHOOK: query: insert overwrite table sih_i_part partition (p='2') select key+10000, value from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@sih_i_part@p=2
+POSTHOOK: Lineage: sih_i_part PARTITION(p=2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: sih_i_part PARTITION(p=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: insert overwrite table sih_i_part partition (p='3') select key+20000, value from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@sih_i_part@p=3
+POSTHOOK: query: insert overwrite table sih_i_part partition (p='3') select key+20000, value from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@sih_i_part@p=3
+POSTHOOK: Lineage: sih_i_part PARTITION(p=3).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: sih_i_part PARTITION(p=3).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: create table sih_src as select key, value from sih_i_part order by key, value
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@sih_i_part
+PREHOOK: Input: default@sih_i_part@p=1
+PREHOOK: Input: default@sih_i_part@p=2
+PREHOOK: Input: default@sih_i_part@p=3
+PREHOOK: Output: database:default
+PREHOOK: Output: default@sih_src
+POSTHOOK: query: create table sih_src as select key, value from sih_i_part order by key, value
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@sih_i_part
+POSTHOOK: Input: default@sih_i_part@p=1
+POSTHOOK: Input: default@sih_i_part@p=2
+POSTHOOK: Input: default@sih_i_part@p=3
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@sih_src
+POSTHOOK: Lineage: sih_src.key SIMPLE [(sih_i_part)sih_i_part.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: sih_src.value SIMPLE [(sih_i_part)sih_i_part.FieldSchema(name:value, type:string, comment:null), ]
+PREHOOK: query: create table sih_src2 as select key, value from sih_src order by key, value
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@sih_src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@sih_src2
+POSTHOOK: query: create table sih_src2 as select key, value from sih_src order by key, value
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@sih_src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@sih_src2
+POSTHOOK: Lineage: sih_src2.key SIMPLE [(sih_src)sih_src.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: sih_src2.value SIMPLE [(sih_src)sih_src.FieldSchema(name:value, type:string, comment:null), ]
+PREHOOK: query: -- Relaxing hive.exec.mode.local.auto.input.files.max=1.
+-- Hadoop20 will not generate more splits than there are files (one).
+-- Hadoop23 generate splits correctly (four), hence the max needs to be adjusted to ensure running in local mode.
+-- Default value is hive.exec.mode.local.auto.input.files.max=4 which produces expected behavior on Hadoop23.
+-- hive.sample.seednumber is required because Hadoop23 generates multiple splits and tablesample is non-repeatable without it.
+
+-- sample split, running locally limited by num tasks
+
+desc formatted sih_src
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@sih_src
+# col_name data_type comment
+
+key int
+value string
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+#### A masked pattern was here ####
+Table Type: MANAGED_TABLE
+Table Parameters:
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ numFiles 1
+ numRows 1500
+ rawDataSize 18124
+ totalSize 19624
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
+PREHOOK: query: explain select count(1) from sih_src
+PREHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: 1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(1) from sih_src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@sih_src
+#### A masked pattern was here ####
+1500
+PREHOOK: query: explain select count(1) from sih_src tablesample(1 percent)
+PREHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: sih_src
+ Statistics: Num rows: 1500 Data size: 18124 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ Statistics: Num rows: 1500 Data size: 18124 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: count(1)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: bigint)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(1) from sih_src tablesample(1 percent)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@sih_src
+#### A masked pattern was here ####
+25
+PREHOOK: query: explain select count(1) from sih_src tablesample(10 rows)
+PREHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: sih_src
+ Row Limit Per Split: 10
+ Statistics: Num rows: 1500 Data size: 18124 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ Statistics: Num rows: 1500 Data size: 18124 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: count(1)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: bigint)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(1) from sih_src tablesample(10 rows)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@sih_src
+#### A masked pattern was here ####
+650
[32/50] [abbrv] hive git commit: HIVE-13111: Fix timestamp /
interval_day_time wrong results with HIVE-9862 (Matt McCline,
reviewed by Jason Dere)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticDateScalarBase.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticDateScalarBase.txt b/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticDateScalarBase.txt
deleted file mode 100644
index c2ddd67..0000000
--- a/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticDateScalarBase.txt
+++ /dev/null
@@ -1,126 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
-
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
-import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil;
-import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
-import org.apache.hadoop.hive.ql.exec.vector.*;
-import org.apache.hadoop.hive.serde2.io.DateWritable;
-
-/**
- * Generated from template TimestampColumnArithmeticDateScalarBase.txt, which covers binary arithmetic
- * expressions between a column and a scalar.
- */
-public abstract class <BaseClassName> extends VectorExpression {
-
- private static final long serialVersionUID = 1L;
-
- private int colNum;
- private PisaTimestamp value;
- private int outputColumn;
- private PisaTimestamp scratchPisaTimestamp;
-
- public <BaseClassName>(int colNum, long value, int outputColumn) {
- this.colNum = colNum;
- this.value = new PisaTimestamp().updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) value));
- this.outputColumn = outputColumn;
- scratchPisaTimestamp = new PisaTimestamp();
- }
-
- public <BaseClassName>() {
- }
-
- @Override
- public void evaluate(VectorizedRowBatch batch) {
-
- if (childExpressions != null) {
- super.evaluateChildren(batch);
- }
-
- // Input #1 is type Timestamp (PisaTimestamp).
- TimestampColumnVector inputColVector1 = (TimestampColumnVector) batch.cols[colNum];
-
- // Output is type Timestamp.
- TimestampColumnVector outputColVector = (TimestampColumnVector) batch.cols[outputColumn];
-
- int[] sel = batch.selected;
- boolean[] inputIsNull = inputColVector1.isNull;
- boolean[] outputIsNull = outputColVector.isNull;
- outputColVector.noNulls = inputColVector1.noNulls;
- outputColVector.isRepeating = inputColVector1.isRepeating;
- int n = batch.size;
-
- // return immediately if batch is empty
- if (n == 0) {
- return;
- }
-
- if (inputColVector1.isRepeating) {
- outputColVector.<OperatorMethod>(
- inputColVector1.asScratchPisaTimestamp(0), value, 0);
-
- // Even if there are no nulls, we always copy over entry 0. Simplifies code.
- outputIsNull[0] = inputIsNull[0];
- } else if (inputColVector1.noNulls) {
- if (batch.selectedInUse) {
- for(int j = 0; j != n; j++) {
- int i = sel[j];
- outputColVector.<OperatorMethod>(
- inputColVector1.asScratchPisaTimestamp(i), value, i);
- }
- } else {
- for(int i = 0; i != n; i++) {
- outputColVector.<OperatorMethod>(
- inputColVector1.asScratchPisaTimestamp(i), value, i);
- }
- }
- } else /* there are nulls */ {
- if (batch.selectedInUse) {
- for(int j = 0; j != n; j++) {
- int i = sel[j];
- outputColVector.<OperatorMethod>(
- inputColVector1.asScratchPisaTimestamp(i), value, i);
- outputIsNull[i] = inputIsNull[i];
- }
- } else {
- for(int i = 0; i != n; i++) {
- outputColVector.<OperatorMethod>(
- inputColVector1.asScratchPisaTimestamp(i), value, i);
- }
- System.arraycopy(inputIsNull, 0, outputIsNull, 0, n);
- }
- }
-
- NullUtil.setNullOutputEntriesColScalar(outputColVector, batch.selectedInUse, sel, n);
- }
-
- @Override
- public int getOutputColumn() {
- return outputColumn;
- }
-
- @Override
- public String getOutputType() {
- return "timestamp";
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticIntervalYearMonthColumn.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticIntervalYearMonthColumn.txt b/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticIntervalYearMonthColumn.txt
index 2f33920..4ac2174 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticIntervalYearMonthColumn.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticIntervalYearMonthColumn.txt
@@ -18,7 +18,9 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
+import java.sql.Timestamp;
+
+import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil;
import org.apache.hadoop.hive.ql.exec.vector.*;
@@ -37,14 +39,14 @@ public class <ClassName> extends VectorExpression {
private int colNum1;
private int colNum2;
private int outputColumn;
- private PisaTimestamp scratchPisaTimestamp;
+ private HiveIntervalYearMonth scratchIntervalYearMonth2;
private DateTimeMath dtm = new DateTimeMath();
public <ClassName>(int colNum1, int colNum2, int outputColumn) {
this.colNum1 = colNum1;
this.colNum2 = colNum2;
this.outputColumn = outputColumn;
- scratchPisaTimestamp = new PisaTimestamp();
+ scratchIntervalYearMonth2 = new HiveIntervalYearMonth();
}
public <ClassName>() {
@@ -57,7 +59,7 @@ public class <ClassName> extends VectorExpression {
super.evaluateChildren(batch);
}
- // Input #1 is type Timestamp (PisaTimestamp).
+ // Input #1 is type Timestamp.
TimestampColumnVector inputColVector1 = (TimestampColumnVector) batch.cols[colNum1];
// Input #2 is type Interval_Year_Month (months).
@@ -91,52 +93,59 @@ public class <ClassName> extends VectorExpression {
* conditional checks in the inner loop.
*/
if (inputColVector1.isRepeating && inputColVector2.isRepeating) {
- outputColVector.set(0,
- dtm.addMonthsToPisaTimestamp(inputColVector1.asScratchPisaTimestamp(0), <OperatorSymbol> (int) vector2[0],
- scratchPisaTimestamp));
+ scratchIntervalYearMonth2.set((int) vector2[0]);
+ dtm.<OperatorMethod>(
+ inputColVector1.asScratchTimestamp(0), scratchIntervalYearMonth2, outputColVector.getScratchTimestamp());
+ outputColVector.setFromScratchTimestamp(0);
} else if (inputColVector1.isRepeating) {
+ Timestamp value1 = inputColVector1.asScratchTimestamp(0);
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputColVector.set(i,
- dtm.addMonthsToPisaTimestamp(inputColVector1.asScratchPisaTimestamp(0), <OperatorSymbol> (int) vector2[i],
- scratchPisaTimestamp));
+ scratchIntervalYearMonth2.set((int) vector2[i]);
+ dtm.<OperatorMethod>(
+ value1, scratchIntervalYearMonth2, outputColVector.getScratchTimestamp());
+ outputColVector.setFromScratchTimestamp(i);
}
} else {
for(int i = 0; i != n; i++) {
- outputColVector.set(i,
- dtm.addMonthsToPisaTimestamp(inputColVector1.asScratchPisaTimestamp(0), <OperatorSymbol> (int) vector2[i],
- scratchPisaTimestamp));
+ scratchIntervalYearMonth2.set((int) vector2[i]);
+ dtm.<OperatorMethod>(
+ value1, scratchIntervalYearMonth2, outputColVector.getScratchTimestamp());
+ outputColVector.setFromScratchTimestamp(i);
}
}
} else if (inputColVector2.isRepeating) {
+ scratchIntervalYearMonth2.set((int) vector2[0]);
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputColVector.set(i,
- dtm.addMonthsToPisaTimestamp(inputColVector1.asScratchPisaTimestamp(i), <OperatorSymbol> (int) vector2[0],
- scratchPisaTimestamp));
+ dtm.<OperatorMethod>(
+ inputColVector1.asScratchTimestamp(i), scratchIntervalYearMonth2, outputColVector.getScratchTimestamp());
+ outputColVector.setFromScratchTimestamp(i);
}
} else {
for(int i = 0; i != n; i++) {
- outputColVector.set(i,
- dtm.addMonthsToPisaTimestamp(inputColVector1.asScratchPisaTimestamp(i), <OperatorSymbol> (int) vector2[0],
- scratchPisaTimestamp));
+ dtm.<OperatorMethod>(
+ inputColVector1.asScratchTimestamp(i), scratchIntervalYearMonth2, outputColVector.getScratchTimestamp());
+ outputColVector.setFromScratchTimestamp(i);
}
}
} else {
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputColVector.set(i,
- dtm.addMonthsToPisaTimestamp(inputColVector1.asScratchPisaTimestamp(i), <OperatorSymbol> (int) vector2[i],
- scratchPisaTimestamp));
+ scratchIntervalYearMonth2.set((int) vector2[i]);
+ dtm.<OperatorMethod>(
+ inputColVector1.asScratchTimestamp(i), scratchIntervalYearMonth2, outputColVector.getScratchTimestamp());
+ outputColVector.setFromScratchTimestamp(i);
}
} else {
for(int i = 0; i != n; i++) {
- outputColVector.set(i,
- dtm.addMonthsToPisaTimestamp(inputColVector1.asScratchPisaTimestamp(i), <OperatorSymbol> (int) vector2[i],
- scratchPisaTimestamp));
+ scratchIntervalYearMonth2.set((int) vector2[i]);
+ dtm.<OperatorMethod>(
+ inputColVector1.asScratchTimestamp(i), scratchIntervalYearMonth2, outputColVector.getScratchTimestamp());
+ outputColVector.setFromScratchTimestamp(i);
}
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticIntervalYearMonthScalar.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticIntervalYearMonthScalar.txt b/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticIntervalYearMonthScalar.txt
index 9f5c24e..9382aca 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticIntervalYearMonthScalar.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticIntervalYearMonthScalar.txt
@@ -18,7 +18,7 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
+import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
@@ -36,16 +36,14 @@ public class <ClassName> extends VectorExpression {
private static final long serialVersionUID = 1L;
private int colNum;
- private long value;
+ private HiveIntervalYearMonth value;
private int outputColumn;
- private PisaTimestamp scratchPisaTimestamp;
private DateTimeMath dtm = new DateTimeMath();
public <ClassName>(int colNum, long value, int outputColumn) {
this.colNum = colNum;
- this.value = value;
+ this.value = new HiveIntervalYearMonth((int) value);
this.outputColumn = outputColumn;
- scratchPisaTimestamp = new PisaTimestamp();
}
public <ClassName>() {
@@ -58,7 +56,7 @@ public class <ClassName> extends VectorExpression {
super.evaluateChildren(batch);
}
- // Input #1 is type Timestamp (PisaTimestamp).
+ // Input #1 is type Timestamp.
TimestampColumnVector inputColVector1 = (TimestampColumnVector) batch.cols[colNum];
// Output is type Timestamp.
@@ -77,41 +75,40 @@ public class <ClassName> extends VectorExpression {
}
if (inputColVector1.isRepeating) {
- outputColVector.set(0,
- dtm.addMonthsToPisaTimestamp(inputColVector1.asScratchPisaTimestamp(0), <OperatorSymbol> (int) value,
- scratchPisaTimestamp));
-
+ dtm.<OperatorMethod>(
+ inputColVector1.asScratchTimestamp(0), value, outputColVector.getScratchTimestamp());
+ outputColVector.setFromScratchTimestamp(0);
// Even if there are no nulls, we always copy over entry 0. Simplifies code.
outputIsNull[0] = inputIsNull[0];
} else if (inputColVector1.noNulls) {
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputColVector.set(i,
- dtm.addMonthsToPisaTimestamp(inputColVector1.asScratchPisaTimestamp(i), <OperatorSymbol> (int) value,
- scratchPisaTimestamp));
+ dtm.<OperatorMethod>(
+ inputColVector1.asScratchTimestamp(i), value, outputColVector.getScratchTimestamp());
+ outputColVector.setFromScratchTimestamp(i);
}
} else {
for(int i = 0; i != n; i++) {
- outputColVector.set(i,
- dtm.addMonthsToPisaTimestamp(inputColVector1.asScratchPisaTimestamp(i), <OperatorSymbol> (int) value,
- scratchPisaTimestamp));
+ dtm.<OperatorMethod>(
+ inputColVector1.asScratchTimestamp(i), value, outputColVector.getScratchTimestamp());
+ outputColVector.setFromScratchTimestamp(i);
}
}
} else /* there are nulls */ {
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputColVector.set(i,
- dtm.addMonthsToPisaTimestamp(inputColVector1.asScratchPisaTimestamp(i), <OperatorSymbol> (int) value,
- scratchPisaTimestamp));
+ dtm.<OperatorMethod>(
+ inputColVector1.asScratchTimestamp(i), value, outputColVector.getScratchTimestamp());
+ outputColVector.setFromScratchTimestamp(i);
outputIsNull[i] = inputIsNull[i];
}
} else {
for(int i = 0; i != n; i++) {
- outputColVector.set(i,
- dtm.addMonthsToPisaTimestamp(inputColVector1.asScratchPisaTimestamp(i), <OperatorSymbol> (int) value,
- scratchPisaTimestamp));
+ dtm.<OperatorMethod>(
+ inputColVector1.asScratchTimestamp(i), value, outputColVector.getScratchTimestamp());
+ outputColVector.setFromScratchTimestamp(i);
}
System.arraycopy(inputIsNull, 0, outputIsNull, 0, n);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticTimestampColumn.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticTimestampColumn.txt b/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticTimestampColumn.txt
index dfd45ab..5eaa450 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticTimestampColumn.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticTimestampColumn.txt
@@ -18,7 +18,9 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
+import java.sql.Timestamp;
+
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil;
import org.apache.hadoop.hive.ql.exec.vector.*;
@@ -27,19 +29,135 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
import org.apache.hadoop.hive.ql.util.DateTimeMath;
/**
- * Generated from template TimestampColumnArithmeticTimestampColumn.txt, which covers binary arithmetic
+ * Generated from template TimestampColumnArithmeticTimestampColumnBase.txt, which covers binary arithmetic
* expressions between columns.
*/
-public class <ClassName> extends <BaseClassName> {
+public class <ClassName> extends VectorExpression {
private static final long serialVersionUID = 1L;
+ private int colNum1;
+ private int colNum2;
+ private int outputColumn;
+ private DateTimeMath dtm = new DateTimeMath();
+
public <ClassName>(int colNum1, int colNum2, int outputColumn) {
- super(colNum1, colNum2, outputColumn);
+ this.colNum1 = colNum1;
+ this.colNum2 = colNum2;
+ this.outputColumn = outputColumn;
}
public <ClassName>() {
- super();
+ }
+
+ @Override
+ public void evaluate(VectorizedRowBatch batch) {
+
+ if (childExpressions != null) {
+ super.evaluateChildren(batch);
+ }
+
+ // Input #1 is type <OperandType1>.
+ <InputColumnVectorType1> inputColVector1 = (<InputColumnVectorType1>) batch.cols[colNum1];
+
+ // Input #2 is type <OperandType2>.
+ <InputColumnVectorType2> inputColVector2 = (<InputColumnVectorType2>) batch.cols[colNum2];
+
+ // Output is type <ReturnType>.
+ <OutputColumnVectorType> outputColVector = (<OutputColumnVectorType>) batch.cols[outputColumn];
+
+ int[] sel = batch.selected;
+ int n = batch.size;
+
+ // return immediately if batch is empty
+ if (n == 0) {
+ return;
+ }
+
+ outputColVector.isRepeating =
+ inputColVector1.isRepeating && inputColVector2.isRepeating
+ || inputColVector1.isRepeating && !inputColVector1.noNulls && inputColVector1.isNull[0]
+ || inputColVector2.isRepeating && !inputColVector2.noNulls && inputColVector2.isNull[0];
+
+ // Handle nulls first
+ NullUtil.propagateNullsColCol(
+ inputColVector1, inputColVector2, outputColVector, sel, n, batch.selectedInUse);
+
+ /* Disregard nulls for processing. In other words,
+ * the arithmetic operation is performed even if one or
+ * more inputs are null. This is to improve speed by avoiding
+ * conditional checks in the inner loop.
+ */
+ if (inputColVector1.isRepeating && inputColVector2.isRepeating) {
+ dtm.<OperatorMethod>(
+ inputColVector1.asScratch<CamelOperandType1>(0), inputColVector2.asScratch<CamelOperandType2>(0), outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(0);
+ } else if (inputColVector1.isRepeating) {
+ <HiveOperandType1> value1 = inputColVector1.asScratch<CamelOperandType1>(0);
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ dtm.<OperatorMethod>(
+ value1, inputColVector2.asScratch<CamelOperandType2>(i), outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ dtm.<OperatorMethod>(
+ value1, inputColVector2.asScratch<CamelOperandType2>(i), outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ }
+ }
+ } else if (inputColVector2.isRepeating) {
+ <HiveOperandType2> value2 = inputColVector2.asScratch<CamelOperandType2>(0);
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ dtm.<OperatorMethod>(
+ inputColVector1.asScratch<CamelOperandType1>(i), value2, outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ dtm.<OperatorMethod>(
+ inputColVector1.asScratch<CamelOperandType1>(i), value2, outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ }
+ }
+ } else {
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ dtm.<OperatorMethod>(
+ inputColVector1.asScratch<CamelOperandType1>(i), inputColVector2.asScratch<CamelOperandType2>(i), outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ dtm.<OperatorMethod>(
+ inputColVector1.asScratch<CamelOperandType1>(i), inputColVector2.asScratch<CamelOperandType2>(i), outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ }
+ }
+ }
+
+ /* For the case when the output can have null values, follow
+ * the convention that the data values must be 1 for long and
+ * NaN for double. This is to prevent possible later zero-divide errors
+ * in complex arithmetic expressions like col2 / (col1 - 1)
+ * in the case when some col1 entries are null.
+ */
+ NullUtil.setNullDataEntries<CamelReturnType>(outputColVector, batch.selectedInUse, sel, n);
+ }
+
+ @Override
+ public int getOutputColumn() {
+ return outputColumn;
+ }
+
+ @Override
+ public String getOutputType() {
+ return "<ReturnType>";
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticTimestampColumnBase.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticTimestampColumnBase.txt b/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticTimestampColumnBase.txt
deleted file mode 100644
index 0e52f6c..0000000
--- a/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticTimestampColumnBase.txt
+++ /dev/null
@@ -1,152 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
-
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil;
-import org.apache.hadoop.hive.ql.exec.vector.*;
-import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
-import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
-import org.apache.hadoop.hive.ql.util.DateTimeMath;
-
-/**
- * Generated from template TimestampColumnArithmeticTimestampColumnBase.txt, which covers binary arithmetic
- * expressions between columns.
- */
-public abstract class <BaseClassName> extends VectorExpression {
-
- private static final long serialVersionUID = 1L;
-
- private int colNum1;
- private int colNum2;
- private int outputColumn;
- private DateTimeMath dtm = new DateTimeMath();
-
- public <BaseClassName>(int colNum1, int colNum2, int outputColumn) {
- this.colNum1 = colNum1;
- this.colNum2 = colNum2;
- this.outputColumn = outputColumn;
- }
-
- public <BaseClassName>() {
- }
-
- @Override
- public void evaluate(VectorizedRowBatch batch) {
-
- if (childExpressions != null) {
- super.evaluateChildren(batch);
- }
-
- // Input #1 is type timestamp/interval_day_time (PisaTimestamp).
- TimestampColumnVector inputColVector1 = (TimestampColumnVector) batch.cols[colNum1];
-
- // Input #2 is type timestamp/interval_day_time (PisaTimestamp).
- TimestampColumnVector inputColVector2 = (TimestampColumnVector) batch.cols[colNum2];
-
- // Output is type timestamp/interval_day_time (PisaTimestamp).
- TimestampColumnVector outputColVector = (TimestampColumnVector) batch.cols[outputColumn];
-
- int[] sel = batch.selected;
- int n = batch.size;
-
- // return immediately if batch is empty
- if (n == 0) {
- return;
- }
-
- outputColVector.isRepeating =
- inputColVector1.isRepeating && inputColVector2.isRepeating
- || inputColVector1.isRepeating && !inputColVector1.noNulls && inputColVector1.isNull[0]
- || inputColVector2.isRepeating && !inputColVector2.noNulls && inputColVector2.isNull[0];
-
- // Handle nulls first
- NullUtil.propagateNullsColCol(
- inputColVector1, inputColVector2, outputColVector, sel, n, batch.selectedInUse);
-
- /* Disregard nulls for processing. In other words,
- * the arithmetic operation is performed even if one or
- * more inputs are null. This is to improve speed by avoiding
- * conditional checks in the inner loop.
- */
- if (inputColVector1.isRepeating && inputColVector2.isRepeating) {
- outputColVector.<OperatorMethod>(
- inputColVector1.asScratchPisaTimestamp(0), inputColVector2.asScratchPisaTimestamp(0), 0);
- } else if (inputColVector1.isRepeating) {
- if (batch.selectedInUse) {
- for(int j = 0; j != n; j++) {
- int i = sel[j];
- outputColVector.<OperatorMethod>(
- inputColVector1.asScratchPisaTimestamp(0), inputColVector2.asScratchPisaTimestamp(i), i);
- }
- } else {
- for(int i = 0; i != n; i++) {
- outputColVector.<OperatorMethod>(
- inputColVector1.asScratchPisaTimestamp(0), inputColVector2.asScratchPisaTimestamp(i), i);
- }
- }
- } else if (inputColVector2.isRepeating) {
- if (batch.selectedInUse) {
- for(int j = 0; j != n; j++) {
- int i = sel[j];
- outputColVector.<OperatorMethod>(
- inputColVector1.asScratchPisaTimestamp(i), inputColVector2.asScratchPisaTimestamp(0), i);
- }
- } else {
- for(int i = 0; i != n; i++) {
- outputColVector.<OperatorMethod>(
- inputColVector1.asScratchPisaTimestamp(i), inputColVector2.asScratchPisaTimestamp(0), i);
- }
- }
- } else {
- if (batch.selectedInUse) {
- for(int j = 0; j != n; j++) {
- int i = sel[j];
- outputColVector.<OperatorMethod>(
- inputColVector1.asScratchPisaTimestamp(i), inputColVector2.asScratchPisaTimestamp(i), i);
- }
- } else {
- for(int i = 0; i != n; i++) {
- outputColVector.<OperatorMethod>(
- inputColVector1.asScratchPisaTimestamp(i), inputColVector2.asScratchPisaTimestamp(i), i);
- }
- }
- }
-
- /* For the case when the output can have null values, follow
- * the convention that the data values must be 1 for long and
- * NaN for double. This is to prevent possible later zero-divide errors
- * in complex arithmetic expressions like col2 / (col1 - 1)
- * in the case when some col1 entries are null.
- */
- NullUtil.setNullDataEntriesTimestamp(outputColVector, batch.selectedInUse, sel, n);
- }
-
- @Override
- public int getOutputColumn() {
- return outputColumn;
- }
-
- @Override
- public String getOutputType() {
- return "timestamp";
- }
-}
-
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticTimestampScalar.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticTimestampScalar.txt b/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticTimestampScalar.txt
index f8004ff..c6c872f 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticTimestampScalar.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticTimestampScalar.txt
@@ -19,10 +19,8 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
import java.sql.Timestamp;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
-import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
-import org.apache.hive.common.util.DateUtils;
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
@@ -35,16 +33,100 @@ import org.apache.hadoop.hive.ql.util.DateTimeMath;
* Generated from template TimestampColumnArithmeticTimestampScalar.txt, which covers binary arithmetic
* expressions between a column and a scalar.
*/
-public class <ClassName> extends <BaseClassName> {
+public class <ClassName> extends VectorExpression {
private static final long serialVersionUID = 1L;
- public <ClassName>(int colNum, <ScalarHiveTimestampType2> value, int outputColumn) {
- super(colNum, <PisaTimestampConversion2>, outputColumn);
+ private int colNum;
+ private <HiveOperandType2> value;
+ private int outputColumn;
+ private DateTimeMath dtm = new DateTimeMath();
+
+ public <ClassName>(int colNum, <HiveOperandType2> value, int outputColumn) {
+ this.colNum = colNum;
+ this.value = value;
+ this.outputColumn = outputColumn;
}
public <ClassName>() {
- super();
+ }
+
+ @Override
+ public void evaluate(VectorizedRowBatch batch) {
+
+ if (childExpressions != null) {
+ super.evaluateChildren(batch);
+ }
+
+ // Input #1 is type <OperandType1>.
+ <InputColumnVectorType1> inputColVector1 = (<InputColumnVectorType1>) batch.cols[colNum];
+
+ // Output is type <ReturnType>.
+ <OutputColumnVectorType> outputColVector = (<OutputColumnVectorType>) batch.cols[outputColumn];
+
+ int[] sel = batch.selected;
+ boolean[] inputIsNull = inputColVector1.isNull;
+ boolean[] outputIsNull = outputColVector.isNull;
+ outputColVector.noNulls = inputColVector1.noNulls;
+ outputColVector.isRepeating = inputColVector1.isRepeating;
+ int n = batch.size;
+
+ // return immediately if batch is empty
+ if (n == 0) {
+ return;
+ }
+
+ if (inputColVector1.isRepeating) {
+ dtm.<OperatorMethod>(
+ inputColVector1.asScratch<CamelOperandType1>(0), value, outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(0);
+ // Even if there are no nulls, we always copy over entry 0. Simplifies code.
+ outputIsNull[0] = inputIsNull[0];
+ } else if (inputColVector1.noNulls) {
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ dtm.<OperatorMethod>(
+ inputColVector1.asScratch<CamelOperandType1>(i), value, outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ dtm.<OperatorMethod>(
+ inputColVector1.asScratch<CamelOperandType1>(i), value, outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ }
+ }
+ } else /* there are nulls */ {
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ dtm.<OperatorMethod>(
+ inputColVector1.asScratch<CamelOperandType1>(i), value, outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ outputIsNull[i] = inputIsNull[i];
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ dtm.<OperatorMethod>(
+ inputColVector1.asScratch<CamelOperandType1>(i), value, outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ }
+ System.arraycopy(inputIsNull, 0, outputIsNull, 0, n);
+ }
+ }
+
+ NullUtil.setNullOutputEntriesColScalar(outputColVector, batch.selectedInUse, sel, n);
+ }
+
+ @Override
+ public int getOutputColumn() {
+ return outputColumn;
+ }
+
+ @Override
+ public String getOutputType() {
+ return "<ReturnType>";
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticTimestampScalarBase.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticTimestampScalarBase.txt b/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticTimestampScalarBase.txt
deleted file mode 100644
index a0de1b3..0000000
--- a/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticTimestampScalarBase.txt
+++ /dev/null
@@ -1,125 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
-
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
-import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil;
-import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
-import org.apache.hadoop.hive.ql.exec.vector.*;
-import org.apache.hadoop.hive.ql.util.DateTimeMath;
-
-/**
- * Generated from template TimestampColumnArithmeticTimestampScalarBase.txt, which covers binary arithmetic
- * expressions between a column and a scalar.
- */
-public abstract class <BaseClassName> extends VectorExpression {
-
- private static final long serialVersionUID = 1L;
-
- private int colNum;
- private PisaTimestamp value;
- private int outputColumn;
- private DateTimeMath dtm = new DateTimeMath();
-
- public <BaseClassName>(int colNum, PisaTimestamp value, int outputColumn) {
- this.colNum = colNum;
- this.value = value;
- this.outputColumn = outputColumn;
- }
-
- public <BaseClassName>() {
- }
-
- @Override
- public void evaluate(VectorizedRowBatch batch) {
-
- if (childExpressions != null) {
- super.evaluateChildren(batch);
- }
-
- // Input #1 is type timestamp/interval_day_time (PisaTimestamp).
- TimestampColumnVector inputColVector1 = (TimestampColumnVector) batch.cols[colNum];
-
- // Output is type timestamp/interval_day_time.
- TimestampColumnVector outputColVector = (TimestampColumnVector) batch.cols[outputColumn];
-
- int[] sel = batch.selected;
- boolean[] inputIsNull = inputColVector1.isNull;
- boolean[] outputIsNull = outputColVector.isNull;
- outputColVector.noNulls = inputColVector1.noNulls;
- outputColVector.isRepeating = inputColVector1.isRepeating;
- int n = batch.size;
-
- // return immediately if batch is empty
- if (n == 0) {
- return;
- }
-
- if (inputColVector1.isRepeating) {
- outputColVector.<OperatorMethod>(
- inputColVector1.asScratchPisaTimestamp(0), value, 0);
-
- // Even if there are no nulls, we always copy over entry 0. Simplifies code.
- outputIsNull[0] = inputIsNull[0];
- } else if (inputColVector1.noNulls) {
- if (batch.selectedInUse) {
- for(int j = 0; j != n; j++) {
- int i = sel[j];
- outputColVector.<OperatorMethod>(
- inputColVector1.asScratchPisaTimestamp(i), value, i);
- }
- } else {
- for(int i = 0; i != n; i++) {
- outputColVector.<OperatorMethod>(
- inputColVector1.asScratchPisaTimestamp(i), value, i);
- }
- }
- } else /* there are nulls */ {
- if (batch.selectedInUse) {
- for(int j = 0; j != n; j++) {
- int i = sel[j];
- outputColVector.<OperatorMethod>(
- inputColVector1.asScratchPisaTimestamp(i), value, i);
- outputIsNull[i] = inputIsNull[i];
- }
- } else {
- for(int i = 0; i != n; i++) {
- outputColVector.<OperatorMethod>(
- inputColVector1.asScratchPisaTimestamp(i), value, i);
- }
- System.arraycopy(inputIsNull, 0, outputIsNull, 0, n);
- }
- }
-
- NullUtil.setNullOutputEntriesColScalar(outputColVector, batch.selectedInUse, sel, n);
- }
-
- @Override
- public int getOutputColumn() {
- return outputColumn;
- }
-
- @Override
- public String getOutputType() {
- return "timestamp";
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareLongDoubleScalar.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareLongDoubleScalar.txt b/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareLongDoubleScalar.txt
index 43321644..e0ae206 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareLongDoubleScalar.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareLongDoubleScalar.txt
@@ -21,6 +21,7 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.*;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareTimestampColumn.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareTimestampColumn.txt b/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareTimestampColumn.txt
index fb82d5e..f9fc425 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareTimestampColumn.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareTimestampColumn.txt
@@ -18,24 +18,128 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
-import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
+import java.sql.Timestamp;
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil;
+import org.apache.hadoop.hive.ql.exec.vector.*;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
/**
- * Generated from template TimestampColumnCompareTimestampColumn.txt, which covers comparison
- * expressions between a datetime/interval column and a scalar of the same type. The boolean output
- * is stored in a separate boolean column.
+ * Generated from template TimestampColumnCompareColumn.txt, which covers comparision
+ * expressions between timestamp columns.
*/
-public class <ClassName> extends <BaseClassName> {
+public class <ClassName> extends VectorExpression {
private static final long serialVersionUID = 1L;
+ private int colNum1;
+ private int colNum2;
+ private int outputColumn;
+
public <ClassName>(int colNum1, int colNum2, int outputColumn) {
- super(colNum1, colNum2, outputColumn);
+ this.colNum1 = colNum1;
+ this.colNum2 = colNum2;
+ this.outputColumn = outputColumn;
}
public <ClassName>() {
- super();
+ }
+
+ @Override
+ public void evaluate(VectorizedRowBatch batch) {
+
+ if (childExpressions != null) {
+ super.evaluateChildren(batch);
+ }
+
+ // Input #1 is type <OperandType>.
+ <InputColumnVectorType> inputColVector1 = (<InputColumnVectorType>) batch.cols[colNum1];
+
+ // Input #2 is type <OperandType>.
+ <InputColumnVectorType> inputColVector2 = (<InputColumnVectorType>) batch.cols[colNum2];
+
+ LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumn];
+ int[] sel = batch.selected;
+ int n = batch.size;
+ long[] outputVector = outputColVector.vector;
+
+ // return immediately if batch is empty
+ if (n == 0) {
+ return;
+ }
+
+ outputColVector.isRepeating =
+ inputColVector1.isRepeating && inputColVector2.isRepeating
+ || inputColVector1.isRepeating && !inputColVector1.noNulls && inputColVector1.isNull[0]
+ || inputColVector2.isRepeating && !inputColVector2.noNulls && inputColVector2.isNull[0];
+
+ // Handle nulls first
+ NullUtil.propagateNullsColCol(
+ inputColVector1, inputColVector2, outputColVector, sel, n, batch.selectedInUse);
+
+ /* Disregard nulls for processing. In other words,
+ * the arithmetic operation is performed even if one or
+ * more inputs are null. This is to improve speed by avoiding
+ * conditional checks in the inner loop.
+ */
+ if (inputColVector1.isRepeating && inputColVector2.isRepeating) {
+ outputVector[0] = inputColVector1.compareTo(0, inputColVector2.asScratch<CamelOperandType>(0)) <OperatorSymbol> 0 ? 1 : 0;
+ } else if (inputColVector1.isRepeating) {
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ outputVector[i] = inputColVector1.compareTo(0, inputColVector2.asScratch<CamelOperandType>(i)) <OperatorSymbol> 0 ? 1 : 0;
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ outputVector[i] = inputColVector1.compareTo(0, inputColVector2.asScratch<CamelOperandType>(i)) <OperatorSymbol> 0 ? 1 : 0;
+ }
+ }
+ } else if (inputColVector2.isRepeating) {
+ <HiveOperandType> value2 = inputColVector2.asScratch<CamelOperandType>(0);
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ outputVector[i] = inputColVector1.compareTo(i, value2) <OperatorSymbol> 0 ? 1 : 0;
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ outputVector[i] = inputColVector1.compareTo(i, value2) <OperatorSymbol> 0 ? 1 : 0;
+ }
+ }
+ } else {
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ outputVector[i] = inputColVector1.compareTo(i, inputColVector2.asScratch<CamelOperandType>(i)) <OperatorSymbol> 0 ? 1 : 0;
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ outputVector[i] = inputColVector1.compareTo(i, inputColVector2.asScratch<CamelOperandType>(i)) <OperatorSymbol> 0 ? 1 : 0;
+ }
+ }
+ }
+
+ /* For the case when the output can have null values, follow
+ * the convention that the data values must be 1 for long and
+ * NaN for double. This is to prevent possible later zero-divide errors
+ * in complex arithmetic expressions like col2 / (col1 - 1)
+ * in the case when some col1 entries are null.
+ */
+ NullUtil.setNullDataEntriesLong(outputColVector, batch.selectedInUse, sel, n);
+ }
+
+ @Override
+ public int getOutputColumn() {
+ return outputColumn;
+ }
+
+ @Override
+ public String getOutputType() {
+ return "long";
}
@Override
@@ -45,8 +149,8 @@ public class <ClassName> extends <BaseClassName> {
VectorExpressionDescriptor.Mode.PROJECTION)
.setNumArguments(2)
.setArgumentTypes(
- VectorExpressionDescriptor.ArgumentType.getType("timestamp"),
- VectorExpressionDescriptor.ArgumentType.getType("timestamp"))
+ VectorExpressionDescriptor.ArgumentType.getType("<OperandType>"),
+ VectorExpressionDescriptor.ArgumentType.getType("<OperandType>"))
.setInputExpressionTypes(
VectorExpressionDescriptor.InputExpressionType.COLUMN,
VectorExpressionDescriptor.InputExpressionType.COLUMN).build();
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareTimestampColumnBase.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareTimestampColumnBase.txt b/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareTimestampColumnBase.txt
deleted file mode 100644
index 302be41..0000000
--- a/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareTimestampColumnBase.txt
+++ /dev/null
@@ -1,140 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
-
-import java.sql.Timestamp;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
-
-import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil;
-import org.apache.hadoop.hive.ql.exec.vector.*;
-import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
-import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
-
-/**
- * Generated from template TimestampColumnCompareColumn.txt, which covers comparision
- * expressions between timestamp columns.
- */
-public abstract class <ClassName> extends VectorExpression {
-
- private static final long serialVersionUID = 1L;
-
- private int colNum1;
- private int colNum2;
- private int outputColumn;
-
- public <ClassName>(int colNum1, int colNum2, int outputColumn) {
- this.colNum1 = colNum1;
- this.colNum2 = colNum2;
- this.outputColumn = outputColumn;
- }
-
- public <ClassName>() {
- }
-
- @Override
- public void evaluate(VectorizedRowBatch batch) {
-
- if (childExpressions != null) {
- super.evaluateChildren(batch);
- }
-
- TimestampColumnVector inputColVector1 = (TimestampColumnVector) batch.cols[colNum1];
- TimestampColumnVector inputColVector2 = (TimestampColumnVector) batch.cols[colNum2];
- LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumn];
- int[] sel = batch.selected;
- int n = batch.size;
- long[] outputVector = outputColVector.vector;
-
- // return immediately if batch is empty
- if (n == 0) {
- return;
- }
-
- outputColVector.isRepeating =
- inputColVector1.isRepeating && inputColVector2.isRepeating
- || inputColVector1.isRepeating && !inputColVector1.noNulls && inputColVector1.isNull[0]
- || inputColVector2.isRepeating && !inputColVector2.noNulls && inputColVector2.isNull[0];
-
- // Handle nulls first
- NullUtil.propagateNullsColCol(
- inputColVector1, inputColVector2, outputColVector, sel, n, batch.selectedInUse);
-
- /* Disregard nulls for processing. In other words,
- * the arithmetic operation is performed even if one or
- * more inputs are null. This is to improve speed by avoiding
- * conditional checks in the inner loop.
- */
- if (inputColVector1.isRepeating && inputColVector2.isRepeating) {
- outputVector[0] = inputColVector1.compareTo(0, inputColVector2.asScratchPisaTimestamp(0)) <OperatorSymbol> 0 ? 1 : 0;
- } else if (inputColVector1.isRepeating) {
- if (batch.selectedInUse) {
- for(int j = 0; j != n; j++) {
- int i = sel[j];
- outputVector[i] = inputColVector1.compareTo(0, inputColVector2.asScratchPisaTimestamp(i)) <OperatorSymbol> 0 ? 1 : 0;
- }
- } else {
- for(int i = 0; i != n; i++) {
- outputVector[i] = inputColVector1.compareTo(0, inputColVector2.asScratchPisaTimestamp(i)) <OperatorSymbol> 0 ? 1 : 0;
- }
- }
- } else if (inputColVector2.isRepeating) {
- PisaTimestamp value2 = inputColVector2.asScratchPisaTimestamp(0);
- if (batch.selectedInUse) {
- for(int j = 0; j != n; j++) {
- int i = sel[j];
- outputVector[i] = inputColVector1.compareTo(i, value2) <OperatorSymbol> 0 ? 1 : 0;
- }
- } else {
- for(int i = 0; i != n; i++) {
- outputVector[i] = inputColVector1.compareTo(i, value2) <OperatorSymbol> 0 ? 1 : 0;
- }
- }
- } else {
- if (batch.selectedInUse) {
- for(int j = 0; j != n; j++) {
- int i = sel[j];
- outputVector[i] = inputColVector1.compareTo(i, inputColVector2.asScratchPisaTimestamp(i)) <OperatorSymbol> 0 ? 1 : 0;
- }
- } else {
- for(int i = 0; i != n; i++) {
- outputVector[i] = inputColVector1.compareTo(i, inputColVector2.asScratchPisaTimestamp(i)) <OperatorSymbol> 0 ? 1 : 0;
- }
- }
- }
-
- /* For the case when the output can have null values, follow
- * the convention that the data values must be 1 for long and
- * NaN for double. This is to prevent possible later zero-divide errors
- * in complex arithmetic expressions like col2 / (col1 - 1)
- * in the case when some col1 entries are null.
- */
- NullUtil.setNullDataEntriesLong(outputColVector, batch.selectedInUse, sel, n);
- }
-
- @Override
- public int getOutputColumn() {
- return outputColumn;
- }
-
- @Override
- public String getOutputType() {
- return "long";
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareTimestampScalar.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareTimestampScalar.txt b/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareTimestampScalar.txt
index 58c3352..90701ec 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareTimestampScalar.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareTimestampScalar.txt
@@ -20,26 +20,116 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
import java.sql.Timestamp;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
-
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
+import org.apache.hadoop.hive.ql.exec.vector.*;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
-
/**
- * Generated from template TimestampColumnCompareTimestampScalar.txt, which covers comparison
- * expressions between a datetime/interval column and a scalar of the same type. The boolean output
- * is stored in a separate boolean column.
+ * Generated from template TimestampColumnCompareTimestampScalar.txt, which covers binary comparison
+ * expressions between a column and a scalar. The boolean output is stored in a
+ * separate boolean column.
*/
-public class <ClassName> extends <BaseClassName> {
+public class <ClassName> extends VectorExpression {
private static final long serialVersionUID = 1L;
- public <ClassName>(int colNum, Timestamp value, int outputColumn) {
- super(colNum, new PisaTimestamp(value), outputColumn);
+ private int colNum;
+ private <HiveOperandType> value;
+ private int outputColumn;
+
+ public <ClassName>(int colNum, <HiveOperandType> value, int outputColumn) {
+ this.colNum = colNum;
+ this.value = value;
+ this.outputColumn = outputColumn;
}
public <ClassName>() {
- super();
+ }
+
+ @Override
+ public void evaluate(VectorizedRowBatch batch) {
+
+ if (childExpressions != null) {
+ super.evaluateChildren(batch);
+ }
+
+ // Input #1 is type <OperandType>.
+ <InputColumnVectorType> inputColVector1 = (<InputColumnVectorType>) batch.cols[colNum];
+
+ LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumn];
+
+ int[] sel = batch.selected;
+ boolean[] nullPos = inputColVector1.isNull;
+ boolean[] outNulls = outputColVector.isNull;
+ int n = batch.size;
+ long[] outputVector = outputColVector.vector;
+
+ // return immediately if batch is empty
+ if (n == 0) {
+ return;
+ }
+
+ outputColVector.isRepeating = false;
+ outputColVector.noNulls = inputColVector1.noNulls;
+ if (inputColVector1.noNulls) {
+ if (inputColVector1.isRepeating) {
+ //All must be selected otherwise size would be zero
+ //Repeating property will not change.
+ outputVector[0] = inputColVector1.compareTo(0, value) <OperatorSymbol> 0 ? 1 : 0;
+ outputColVector.isRepeating = true;
+ } else if (batch.selectedInUse) {
+ for(int j=0; j != n; j++) {
+ int i = sel[j];
+ outputVector[i] = inputColVector1.compareTo(i, value) <OperatorSymbol> 0 ? 1 : 0;
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ outputVector[i] = inputColVector1.compareTo(i, value) <OperatorSymbol> 0 ? 1 : 0;
+ }
+ }
+ } else {
+ if (inputColVector1.isRepeating) {
+ //All must be selected otherwise size would be zero
+ //Repeating property will not change.
+ if (!nullPos[0]) {
+ outputVector[0] = inputColVector1.compareTo(0, value) <OperatorSymbol> 0 ? 1 : 0;
+ outNulls[0] = false;
+ } else {
+ outNulls[0] = true;
+ }
+ outputColVector.isRepeating = true;
+ } else if (batch.selectedInUse) {
+ for(int j=0; j != n; j++) {
+ int i = sel[j];
+ if (!nullPos[i]) {
+ outputVector[i] = inputColVector1.compareTo(i, value) <OperatorSymbol> 0 ? 1 : 0;
+ outNulls[i] = false;
+ } else {
+ //comparison with null is null
+ outNulls[i] = true;
+ }
+ }
+ } else {
+ System.arraycopy(nullPos, 0, outNulls, 0, n);
+ for(int i = 0; i != n; i++) {
+ if (!nullPos[i]) {
+ outputVector[i] = inputColVector1.compareTo(i, value) <OperatorSymbol> 0 ? 1 : 0;
+ }
+ }
+ }
+ }
+ }
+
+ @Override
+ public int getOutputColumn() {
+ return outputColumn;
+ }
+
+ @Override
+ public String getOutputType() {
+ return "long";
}
@Override
@@ -49,8 +139,8 @@ public class <ClassName> extends <BaseClassName> {
VectorExpressionDescriptor.Mode.PROJECTION)
.setNumArguments(2)
.setArgumentTypes(
- VectorExpressionDescriptor.ArgumentType.getType("timestamp"),
- VectorExpressionDescriptor.ArgumentType.getType("timestamp"))
+ VectorExpressionDescriptor.ArgumentType.getType("<OperandType>"),
+ VectorExpressionDescriptor.ArgumentType.getType("<OperandType>"))
.setInputExpressionTypes(
VectorExpressionDescriptor.InputExpressionType.COLUMN,
VectorExpressionDescriptor.InputExpressionType.SCALAR).build();
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareTimestampScalarBase.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareTimestampScalarBase.txt b/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareTimestampScalarBase.txt
deleted file mode 100644
index ce940a4..0000000
--- a/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareTimestampScalarBase.txt
+++ /dev/null
@@ -1,131 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
-
-import java.sql.Timestamp;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
-
-import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
-import org.apache.hadoop.hive.ql.exec.vector.*;
-import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
-import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
-
-/**
- * Generated from template TimestampColumnCompareTimestampScalar.txt, which covers binary comparison
- * expressions between a column and a scalar. The boolean output is stored in a
- * separate boolean column.
- */
-public abstract class <ClassName> extends VectorExpression {
-
- private static final long serialVersionUID = 1L;
-
- private int colNum;
- private PisaTimestamp value;
- private int outputColumn;
-
- public <ClassName>(int colNum, PisaTimestamp value, int outputColumn) {
- this.colNum = colNum;
- this.value = value;
- this.outputColumn = outputColumn;
- }
-
- public <ClassName>() {
- }
-
- @Override
- public void evaluate(VectorizedRowBatch batch) {
-
- if (childExpressions != null) {
- super.evaluateChildren(batch);
- }
-
- TimestampColumnVector inputColVector1 = (TimestampColumnVector) batch.cols[colNum];
- LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumn];
- int[] sel = batch.selected;
- boolean[] nullPos = inputColVector1.isNull;
- boolean[] outNulls = outputColVector.isNull;
- int n = batch.size;
- long[] outputVector = outputColVector.vector;
-
- // return immediately if batch is empty
- if (n == 0) {
- return;
- }
-
- outputColVector.isRepeating = false;
- outputColVector.noNulls = inputColVector1.noNulls;
- if (inputColVector1.noNulls) {
- if (inputColVector1.isRepeating) {
- //All must be selected otherwise size would be zero
- //Repeating property will not change.
- outputVector[0] = inputColVector1.compareTo(0, value) <OperatorSymbol> 0 ? 1 : 0;
- outputColVector.isRepeating = true;
- } else if (batch.selectedInUse) {
- for(int j=0; j != n; j++) {
- int i = sel[j];
- outputVector[i] = inputColVector1.compareTo(i, value) <OperatorSymbol> 0 ? 1 : 0;
- }
- } else {
- for(int i = 0; i != n; i++) {
- outputVector[i] = inputColVector1.compareTo(i, value) <OperatorSymbol> 0 ? 1 : 0;
- }
- }
- } else {
- if (inputColVector1.isRepeating) {
- //All must be selected otherwise size would be zero
- //Repeating property will not change.
- if (!nullPos[0]) {
- outputVector[0] = inputColVector1.compareTo(0, value) <OperatorSymbol> 0 ? 1 : 0;
- outNulls[0] = false;
- } else {
- outNulls[0] = true;
- }
- outputColVector.isRepeating = true;
- } else if (batch.selectedInUse) {
- for(int j=0; j != n; j++) {
- int i = sel[j];
- if (!nullPos[i]) {
- outputVector[i] = inputColVector1.compareTo(i, value) <OperatorSymbol> 0 ? 1 : 0;
- outNulls[i] = false;
- } else {
- //comparison with null is null
- outNulls[i] = true;
- }
- }
- } else {
- System.arraycopy(nullPos, 0, outNulls, 0, n);
- for(int i = 0; i != n; i++) {
- if (!nullPos[i]) {
- outputVector[i] = inputColVector1.compareTo(i, value) <OperatorSymbol> 0 ? 1 : 0;
- }
- }
- }
- }
- }
-
- @Override
- public int getOutputColumn() {
- return outputColumn;
- }
-
- @Override
- public String getOutputType() {
- return "long";
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticDateColumn.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticDateColumn.txt b/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticDateColumn.txt
index 8f89bd4..f958be8 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticDateColumn.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticDateColumn.txt
@@ -19,9 +19,8 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
import java.sql.Timestamp;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
-import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
import org.apache.hadoop.hive.ql.exec.vector.*;
@@ -31,29 +30,129 @@ import org.apache.hadoop.hive.ql.exec.vector.*;
* of these ColumnVector imports may be needed. Listing both of them
* rather than using ....vectorization.*;
*/
-import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil;
import org.apache.hadoop.hive.ql.util.DateTimeMath;
-import org.apache.hive.common.util.DateUtils;
+import org.apache.hadoop.hive.serde2.io.DateWritable;
/**
- * Generated from template TimestampScalarArithmeticDateColumn.txt.
+ * Generated from template TimestampScalarArithmeticDateColumnBase.txt.
* Implements a vectorized arithmetic operator with a scalar on the left and a
* column vector on the right. The result is output to an output column vector.
*/
-public class <ClassName> extends <BaseClassName> {
+public class <ClassName> extends VectorExpression {
private static final long serialVersionUID = 1L;
- public <ClassName>(<ScalarHiveTimestampType1> value, int colNum, int outputColumn) {
- super(<PisaTimestampConversion1>, colNum, outputColumn);
+ private int colNum;
+ private <HiveOperandType1> value;
+ private int outputColumn;
+ private Timestamp scratchTimestamp2;
+ private DateTimeMath dtm = new DateTimeMath();
+
+ public <ClassName>(<HiveOperandType1> value, int colNum, int outputColumn) {
+ this.colNum = colNum;
+ this.value = value;
+ this.outputColumn = outputColumn;
+ scratchTimestamp2 = new Timestamp(0);
}
public <ClassName>() {
}
@Override
+ /**
+ * Method to evaluate scalar-column operation in vectorized fashion.
+ *
+ * @batch a package of rows with each column stored in a vector
+ */
+ public void evaluate(VectorizedRowBatch batch) {
+
+ if (childExpressions != null) {
+ super.evaluateChildren(batch);
+ }
+
+ // Input #2 is type date.
+ LongColumnVector inputColVector2 = (LongColumnVector) batch.cols[colNum];
+
+ // Output is type <ReturnType>.
+ <OutputColumnVectorType> outputColVector = (<OutputColumnVectorType>) batch.cols[outputColumn];
+
+ int[] sel = batch.selected;
+ boolean[] inputIsNull = inputColVector2.isNull;
+ boolean[] outputIsNull = outputColVector.isNull;
+ outputColVector.noNulls = inputColVector2.noNulls;
+ outputColVector.isRepeating = inputColVector2.isRepeating;
+ int n = batch.size;
+
+ long[] vector2 = inputColVector2.vector;
+
+ // return immediately if batch is empty
+ if (n == 0) {
+ return;
+ }
+
+ if (inputColVector2.isRepeating) {
+ scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[0]));
+ dtm.<OperatorMethod>(
+ value, scratchTimestamp2, outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(0);
+ // Even if there are no nulls, we always copy over entry 0. Simplifies code.
+ outputIsNull[0] = inputIsNull[0];
+ } else if (inputColVector2.noNulls) {
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[i]));
+ dtm.<OperatorMethod>(
+ value, scratchTimestamp2, outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[i]));
+ dtm.<OperatorMethod>(
+ value, scratchTimestamp2, outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ }
+ }
+ } else { /* there are nulls */
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[i]));
+ dtm.<OperatorMethod>(
+ value, scratchTimestamp2, outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ outputIsNull[i] = inputIsNull[i];
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[i]));
+ dtm.<OperatorMethod>(
+ value, scratchTimestamp2, outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ }
+ System.arraycopy(inputIsNull, 0, outputIsNull, 0, n);
+ }
+ }
+
+ NullUtil.setNullOutputEntriesColScalar(outputColVector, batch.selectedInUse, sel, n);
+ }
+
+ @Override
+ public int getOutputColumn() {
+ return outputColumn;
+ }
+
+ @Override
+ public String getOutputType() {
+ return "<ReturnType>";
+ }
+
+ @Override
public VectorExpressionDescriptor.Descriptor getDescriptor() {
return (new VectorExpressionDescriptor.Builder())
.setMode(
@@ -61,7 +160,7 @@ public class <ClassName> extends <BaseClassName> {
.setNumArguments(2)
.setArgumentTypes(
VectorExpressionDescriptor.ArgumentType.getType("<OperandType1>"),
- VectorExpressionDescriptor.ArgumentType.getType("<OperandType2>"))
+ VectorExpressionDescriptor.ArgumentType.getType("date"))
.setInputExpressionTypes(
VectorExpressionDescriptor.InputExpressionType.SCALAR,
VectorExpressionDescriptor.InputExpressionType.COLUMN).build();
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticDateColumnBase.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticDateColumnBase.txt b/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticDateColumnBase.txt
deleted file mode 100644
index 94be4f6..0000000
--- a/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticDateColumnBase.txt
+++ /dev/null
@@ -1,151 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
-
-import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
-import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
-import org.apache.hadoop.hive.ql.exec.vector.*;
-
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
-/*
- * Because of the templatized nature of the code, either or both
- * of these ColumnVector imports may be needed. Listing both of them
- * rather than using ....vectorization.*;
- */
-import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil;
-import org.apache.hadoop.hive.serde2.io.DateWritable;
-
-/**
- * Generated from template TimestampScalarArithmeticDateColumnBase.txt.
- * Implements a vectorized arithmetic operator with a scalar on the left and a
- * column vector on the right. The result is output to an output column vector.
- */
-public abstract class <BaseClassName> extends VectorExpression {
-
- private static final long serialVersionUID = 1L;
-
- private int colNum;
- private PisaTimestamp value;
- private int outputColumn;
- private PisaTimestamp scratchPisaTimestamp;
-
- public <BaseClassName>(PisaTimestamp value, int colNum, int outputColumn) {
- this.colNum = colNum;
- this.value = value;
- this.outputColumn = outputColumn;
- scratchPisaTimestamp = new PisaTimestamp();
- }
-
- public <BaseClassName>() {
- }
-
- @Override
- /**
- * Method to evaluate scalar-column operation in vectorized fashion.
- *
- * @batch a package of rows with each column stored in a vector
- */
- public void evaluate(VectorizedRowBatch batch) {
-
- if (childExpressions != null) {
- super.evaluateChildren(batch);
- }
-
- // Input #2 is type date.
- LongColumnVector inputColVector2 = (LongColumnVector) batch.cols[colNum];
-
- // Output is type Timestamp.
- TimestampColumnVector outputColVector = (TimestampColumnVector) batch.cols[outputColumn];
-
- int[] sel = batch.selected;
- boolean[] inputIsNull = inputColVector2.isNull;
- boolean[] outputIsNull = outputColVector.isNull;
- outputColVector.noNulls = inputColVector2.noNulls;
- outputColVector.isRepeating = inputColVector2.isRepeating;
- int n = batch.size;
-
- long[] vector2 = inputColVector2.vector;
-
- // return immediately if batch is empty
- if (n == 0) {
- return;
- }
-
- if (inputColVector2.isRepeating) {
- outputColVector.<OperatorMethod>(
- value,
- scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector2[0])),
- 0);
-
- // Even if there are no nulls, we always copy over entry 0. Simplifies code.
- outputIsNull[0] = inputIsNull[0];
- } else if (inputColVector2.noNulls) {
- if (batch.selectedInUse) {
- for(int j = 0; j != n; j++) {
- int i = sel[j];
- outputColVector.<OperatorMethod>(
- value,
- scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector2[i])),
- i);
- }
- } else {
- for(int i = 0; i != n; i++) {
- outputColVector.<OperatorMethod>(
- value,
- scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector2[i])),
- i);
- }
- }
- } else { /* there are nulls */
- if (batch.selectedInUse) {
- for(int j = 0; j != n; j++) {
- int i = sel[j];
- outputColVector.<OperatorMethod>(
- value,
- scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector2[i])),
- i);
- outputIsNull[i] = inputIsNull[i];
- }
- } else {
- for(int i = 0; i != n; i++) {
- outputColVector.<OperatorMethod>(
- value,
- scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector2[i])),
- i);
- }
- System.arraycopy(inputIsNull, 0, outputIsNull, 0, n);
- }
- }
-
- NullUtil.setNullOutputEntriesColScalar(outputColVector, batch.selectedInUse, sel, n);
- }
-
- @Override
- public int getOutputColumn() {
- return outputColumn;
- }
-
- @Override
- public String getOutputType() {
- return "timestamp";
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticIntervalYearMonthColumn.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticIntervalYearMonthColumn.txt b/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticIntervalYearMonthColumn.txt
index e9b9e67..585027a 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticIntervalYearMonthColumn.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticIntervalYearMonthColumn.txt
@@ -18,11 +18,13 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
+import java.sql.Timestamp;
+
+import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
import org.apache.hadoop.hive.ql.exec.vector.*;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
/*
* Because of the templatized nature of the code, either or both
* of these ColumnVector imports may be needed. Listing both of them
@@ -44,16 +46,16 @@ public class <ClassName> extends VectorExpression {
private static final long serialVersionUID = 1L;
private int colNum;
- private PisaTimestamp value;
+ private Timestamp value;
private int outputColumn;
- private PisaTimestamp scratchPisaTimestamp;
+ private HiveIntervalYearMonth scratchIntervalYearMonth2;
private DateTimeMath dtm = new DateTimeMath();
- public <ClassName>(PisaTimestamp value, int colNum, int outputColumn) {
+ public <ClassName>(Timestamp value, int colNum, int outputColumn) {
this.colNum = colNum;
this.value = value;
this.outputColumn = outputColumn;
- scratchPisaTimestamp = new PisaTimestamp();
+ scratchIntervalYearMonth2 = new HiveIntervalYearMonth();
}
public <ClassName>() {
@@ -72,61 +74,65 @@ public class <ClassName> extends VectorExpression {
}
// Input #2 is type Interval_Year_Month (months).
- LongColumnVector inputColVector = (LongColumnVector) batch.cols[colNum];
+ LongColumnVector inputColVector2 = (LongColumnVector) batch.cols[colNum];
// Output is type Timestamp.
TimestampColumnVector outputColVector = (TimestampColumnVector) batch.cols[outputColumn];
int[] sel = batch.selected;
- boolean[] inputIsNull = inputColVector.isNull;
+ boolean[] inputIsNull = inputColVector2.isNull;
boolean[] outputIsNull = outputColVector.isNull;
- outputColVector.noNulls = inputColVector.noNulls;
- outputColVector.isRepeating = inputColVector.isRepeating;
+ outputColVector.noNulls = inputColVector2.noNulls;
+ outputColVector.isRepeating = inputColVector2.isRepeating;
int n = batch.size;
- long[] vector = inputColVector.vector;
+ long[] vector2 = inputColVector2.vector;
// return immediately if batch is empty
if (n == 0) {
return;
}
- if (inputColVector.isRepeating) {
- outputColVector.set(0,
- dtm.addMonthsToPisaTimestamp(value, <OperatorSymbol> (int) vector[0],
- scratchPisaTimestamp));
-
+ if (inputColVector2.isRepeating) {
+ scratchIntervalYearMonth2.set((int) vector2[0]);
+ dtm.<OperatorMethod>(
+ value, scratchIntervalYearMonth2, outputColVector.getScratchTimestamp());
+ outputColVector.setFromScratchTimestamp(0);
// Even if there are no nulls, we always copy over entry 0. Simplifies code.
outputIsNull[0] = inputIsNull[0];
- } else if (inputColVector.noNulls) {
+ } else if (inputColVector2.noNulls) {
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputColVector.set(i,
- dtm.addMonthsToPisaTimestamp(value, <OperatorSymbol> (int) vector[i],
- scratchPisaTimestamp));
+ scratchIntervalYearMonth2.set((int) vector2[i]);
+ dtm.<OperatorMethod>(
+ value, scratchIntervalYearMonth2, outputColVector.getScratchTimestamp());
+ outputColVector.setFromScratchTimestamp(i);
}
} else {
for(int i = 0; i != n; i++) {
- outputColVector.set(i,
- dtm.addMonthsToPisaTimestamp(value, <OperatorSymbol> (int) vector[i],
- scratchPisaTimestamp));
+ scratchIntervalYearMonth2.set((int) vector2[i]);
+ dtm.<OperatorMethod>(
+ value, scratchIntervalYearMonth2, outputColVector.getScratchTimestamp());
+ outputColVector.setFromScratchTimestamp(i);
}
}
} else { /* there are nulls */
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputColVector.set(i,
- dtm.addMonthsToPisaTimestamp(value, <OperatorSymbol> (int) vector[i],
- scratchPisaTimestamp));
+ scratchIntervalYearMonth2.set((int) vector2[i]);
+ dtm.<OperatorMethod>(
+ value, scratchIntervalYearMonth2, outputColVector.getScratchTimestamp());
+ outputColVector.setFromScratchTimestamp(i);
outputIsNull[i] = inputIsNull[i];
}
} else {
for(int i = 0; i != n; i++) {
- outputColVector.set(i,
- dtm.addMonthsToPisaTimestamp(value, <OperatorSymbol> (int) vector[i],
- scratchPisaTimestamp));
+ scratchIntervalYearMonth2.set((int) vector2[i]);
+ dtm.<OperatorMethod>(
+ value, scratchIntervalYearMonth2, outputColVector.getScratchTimestamp());
+ outputColVector.setFromScratchTimestamp(i);
}
System.arraycopy(inputIsNull, 0, outputIsNull, 0, n);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticTimestampColumn.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticTimestampColumn.txt b/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticTimestampColumn.txt
index 6725908..996c86a 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticTimestampColumn.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticTimestampColumn.txt
@@ -19,10 +19,8 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
import java.sql.Timestamp;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
-import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
-import org.apache.hive.common.util.DateUtils;
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
import org.apache.hadoop.hive.ql.exec.vector.*;
@@ -38,22 +36,113 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil;
import org.apache.hadoop.hive.ql.util.DateTimeMath;
/**
- * Generated from template TimestampScalarArithmeticTimestampColumn.txt.
+ * Generated from template TimestampScalarArithmeticTimestampColumnBase.txt.
* Implements a vectorized arithmetic operator with a scalar on the left and a
* column vector on the right. The result is output to an output column vector.
*/
-public class <ClassName> extends <BaseClassName> {
+public class <ClassName> extends VectorExpression {
private static final long serialVersionUID = 1L;
- public <ClassName>(<ScalarHiveTimestampType1> value, int colNum, int outputColumn) {
- super(<PisaTimestampConversion1>, colNum, outputColumn);
+ private int colNum;
+ private <HiveOperandType1> value;
+ private int outputColumn;
+ private DateTimeMath dtm = new DateTimeMath();
+
+ public <ClassName>(<HiveOperandType1> value, int colNum, int outputColumn) {
+ this.colNum = colNum;
+ this.value = value;
+ this.outputColumn = outputColumn;
}
public <ClassName>() {
}
@Override
+ /**
+ * Method to evaluate scalar-column operation in vectorized fashion.
+ *
+ * @batch a package of rows with each column stored in a vector
+ */
+ public void evaluate(VectorizedRowBatch batch) {
+
+ if (childExpressions != null) {
+ super.evaluateChildren(batch);
+ }
+
+ // Input #2 is type <OperandType2>.
+ <InputColumnVectorType2> inputColVector2 = (<InputColumnVectorType2>) batch.cols[colNum];
+
+ // Output is type <ReturnType>.
+ <OutputColumnVectorType> outputColVector = (<OutputColumnVectorType>) batch.cols[outputColumn];
+
+ int[] sel = batch.selected;
+ boolean[] inputIsNull = inputColVector2.isNull;
+ boolean[] outputIsNull = outputColVector.isNull;
+ outputColVector.noNulls = inputColVector2.noNulls;
+ outputColVector.isRepeating = inputColVector2.isRepeating;
+ int n = batch.size;
+
+ // return immediately if batch is empty
+ if (n == 0) {
+ return;
+ }
+
+ if (inputColVector2.isRepeating) {
+ dtm.<OperatorMethod>(
+ value, inputColVector2.asScratch<CamelOperandType2>(0), outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(0);
+ // Even if there are no nulls, we always copy over entry 0. Simplifies code.
+ outputIsNull[0] = inputIsNull[0];
+ } else if (inputColVector2.noNulls) {
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ dtm.<OperatorMethod>(
+ value, inputColVector2.asScratch<CamelOperandType2>(i), outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ dtm.<OperatorMethod>(
+ value, inputColVector2.asScratch<CamelOperandType2>(i), outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+
+ }
+ }
+ } else { /* there are nulls */
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ dtm.<OperatorMethod>(
+ value, inputColVector2.asScratch<CamelOperandType2>(i), outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ outputIsNull[i] = inputIsNull[i];
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ dtm.<OperatorMethod>(
+ value, inputColVector2.asScratch<CamelOperandType2>(i), outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ }
+ System.arraycopy(inputIsNull, 0, outputIsNull, 0, n);
+ }
+ }
+
+ NullUtil.setNullOutputEntriesColScalar(outputColVector, batch.selectedInUse, sel, n);
+ }
+
+ @Override
+ public int getOutputColumn() {
+ return outputColumn;
+ }
+
+ @Override
+ public String getOutputType() {
+ return "timestamp";
+ }
+
+ @Override
public VectorExpressionDescriptor.Descriptor getDescriptor() {
return (new VectorExpressionDescriptor.Builder())
.setMode(
[11/50] [abbrv] hive git commit: HIVE-12367 : Lock/unlock database
should add current database to inputs and outputs of authz hook (Dapeng Sun
via Ashutosh Chauhan)
Posted by jd...@apache.org.
HIVE-12367 : Lock/unlock database should add current database to inputs and outputs of authz hook (Dapeng Sun via Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6bfec2e9
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6bfec2e9
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6bfec2e9
Branch: refs/heads/llap
Commit: 6bfec2e97c4e434646aa9aeffd98c9939313fa6e
Parents: a71edcf
Author: Dapeng Sun <sd...@apache.org>
Authored: Tue Dec 15 19:39:00 2015 -0800
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Thu Mar 24 19:22:28 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/hive/ql/Driver.java | 20 ++++++++++++++++++++
.../hive/ql/parse/DDLSemanticAnalyzer.java | 15 +++++++++++++--
.../clientnegative/dbtxnmgr_nodblock.q.out | 2 ++
.../clientnegative/dbtxnmgr_nodbunlock.q.out | 2 ++
.../lockneg_query_tbl_in_locked_db.q.out | 6 ++++++
.../lockneg_try_db_lock_conflict.q.out | 6 ++++++
.../lockneg_try_drop_locked_db.q.out | 4 ++++
.../lockneg_try_lock_db_in_use.q.out | 6 ++++++
8 files changed, 59 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/6bfec2e9/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index d7e4ac7..7276e31 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -1416,6 +1416,10 @@ public class Driver implements CommandProcessor {
if (!checkConcurrency()) {
return false;
}
+ // Lock operations themselves don't require the lock.
+ if (isExplicitLockOperation()){
+ return false;
+ }
if (!HiveConf.getBoolVar(conf, ConfVars.HIVE_LOCK_MAPRED_ONLY)) {
return true;
}
@@ -1438,6 +1442,22 @@ public class Driver implements CommandProcessor {
return false;
}
+ private boolean isExplicitLockOperation() {
+ HiveOperation currentOpt = plan.getOperation();
+ if (currentOpt != null) {
+ switch (currentOpt) {
+ case LOCKDB:
+ case UNLOCKDB:
+ case LOCKTABLE:
+ case UNLOCKTABLE:
+ return true;
+ default:
+ return false;
+ }
+ }
+ return false;
+ }
+
private CommandProcessorResponse createProcessorResponse(int ret) {
queryDisplay.setErrorMessage(errorMessage);
return new CommandProcessorResponse(ret, errorMessage, SQLState, downstreamError);
http://git-wip-us.apache.org/repos/asf/hive/blob/6bfec2e9/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
index 0c087ed..fe9b8cc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
@@ -2444,8 +2444,12 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
String dbName = unescapeIdentifier(ast.getChild(0).getText());
String mode = unescapeIdentifier(ast.getChild(1).getText().toUpperCase());
- //inputs.add(new ReadEntity(dbName));
- //outputs.add(new WriteEntity(dbName));
+ inputs.add(new ReadEntity(getDatabase(dbName)));
+ // Lock database operation is to acquire the lock explicitly, the operation
+ // itself doesn't need to be locked. Set the WriteEntity as WriteType:
+ // DDL_NO_LOCK here, otherwise it will conflict with Hive's transaction.
+ outputs.add(new WriteEntity(getDatabase(dbName), WriteType.DDL_NO_LOCK));
+
LockDatabaseDesc lockDatabaseDesc = new LockDatabaseDesc(dbName, mode,
HiveConf.getVar(conf, ConfVars.HIVEQUERYID));
lockDatabaseDesc.setQueryStr(ctx.getCmd());
@@ -2457,6 +2461,13 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
private void analyzeUnlockDatabase(ASTNode ast) throws SemanticException {
String dbName = unescapeIdentifier(ast.getChild(0).getText());
+ inputs.add(new ReadEntity(getDatabase(dbName)));
+ // Unlock database operation is to release the lock explicitly, the
+ // operation itself don't need to be locked. Set the WriteEntity as
+ // WriteType: DDL_NO_LOCK here, otherwise it will conflict with
+ // Hive's transaction.
+ outputs.add(new WriteEntity(getDatabase(dbName), WriteType.DDL_NO_LOCK));
+
UnlockDatabaseDesc unlockDatabaseDesc = new UnlockDatabaseDesc(dbName);
DDLWork work = new DDLWork(getInputs(), getOutputs(), unlockDatabaseDesc);
rootTasks.add(TaskFactory.get(work, conf));
http://git-wip-us.apache.org/repos/asf/hive/blob/6bfec2e9/ql/src/test/results/clientnegative/dbtxnmgr_nodblock.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/dbtxnmgr_nodblock.q.out b/ql/src/test/results/clientnegative/dbtxnmgr_nodblock.q.out
index b8bbe93..e783251 100644
--- a/ql/src/test/results/clientnegative/dbtxnmgr_nodblock.q.out
+++ b/ql/src/test/results/clientnegative/dbtxnmgr_nodblock.q.out
@@ -10,4 +10,6 @@ POSTHOOK: type: CREATEDATABASE
POSTHOOK: Output: database:drop_nodblock
PREHOOK: query: lock database drop_nodblock shared
PREHOOK: type: LOCKDATABASE
+PREHOOK: Input: database:drop_nodblock
+PREHOOK: Output: database:drop_nodblock
FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Current transaction manager does not support explicit lock requests. Transaction manager: org.apache.hadoop.hive.ql.lockmgr.DbTxnManager
http://git-wip-us.apache.org/repos/asf/hive/blob/6bfec2e9/ql/src/test/results/clientnegative/dbtxnmgr_nodbunlock.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/dbtxnmgr_nodbunlock.q.out b/ql/src/test/results/clientnegative/dbtxnmgr_nodbunlock.q.out
index ea7a3eb..d7a39f0 100644
--- a/ql/src/test/results/clientnegative/dbtxnmgr_nodbunlock.q.out
+++ b/ql/src/test/results/clientnegative/dbtxnmgr_nodbunlock.q.out
@@ -10,4 +10,6 @@ POSTHOOK: type: CREATEDATABASE
POSTHOOK: Output: database:drop_nodbunlock
PREHOOK: query: unlock database drop_nodbunlock
PREHOOK: type: UNLOCKDATABASE
+PREHOOK: Input: database:drop_nodbunlock
+PREHOOK: Output: database:drop_nodbunlock
FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Current transaction manager does not support explicit lock requests. Transaction manager: org.apache.hadoop.hive.ql.lockmgr.DbTxnManager
http://git-wip-us.apache.org/repos/asf/hive/blob/6bfec2e9/ql/src/test/results/clientnegative/lockneg_query_tbl_in_locked_db.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/lockneg_query_tbl_in_locked_db.q.out b/ql/src/test/results/clientnegative/lockneg_query_tbl_in_locked_db.q.out
index 134a57b..2c46159 100644
--- a/ql/src/test/results/clientnegative/lockneg_query_tbl_in_locked_db.q.out
+++ b/ql/src/test/results/clientnegative/lockneg_query_tbl_in_locked_db.q.out
@@ -34,8 +34,12 @@ POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpar
POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
PREHOOK: query: lock database lockneg1 shared
PREHOOK: type: LOCKDATABASE
+PREHOOK: Input: database:lockneg1
+PREHOOK: Output: database:lockneg1
POSTHOOK: query: lock database lockneg1 shared
POSTHOOK: type: LOCKDATABASE
+POSTHOOK: Input: database:lockneg1
+POSTHOOK: Output: database:lockneg1
PREHOOK: query: show locks database lockneg1
PREHOOK: type: SHOWLOCKS
POSTHOOK: query: show locks database lockneg1
@@ -53,4 +57,6 @@ POSTHOOK: Input: lockneg1@tstsrcpart@ds=2008-04-08/hr=11
500
PREHOOK: query: unlock database lockneg1
PREHOOK: type: UNLOCKDATABASE
+PREHOOK: Input: database:lockneg1
+PREHOOK: Output: database:lockneg1
FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Database lockneg1 is not locked
http://git-wip-us.apache.org/repos/asf/hive/blob/6bfec2e9/ql/src/test/results/clientnegative/lockneg_try_db_lock_conflict.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/lockneg_try_db_lock_conflict.q.out b/ql/src/test/results/clientnegative/lockneg_try_db_lock_conflict.q.out
index 0a5d98a..fe12d83 100644
--- a/ql/src/test/results/clientnegative/lockneg_try_db_lock_conflict.q.out
+++ b/ql/src/test/results/clientnegative/lockneg_try_db_lock_conflict.q.out
@@ -6,9 +6,15 @@ POSTHOOK: type: CREATEDATABASE
POSTHOOK: Output: database:lockneg4
PREHOOK: query: lock database lockneg4 exclusive
PREHOOK: type: LOCKDATABASE
+PREHOOK: Input: database:lockneg4
+PREHOOK: Output: database:lockneg4
POSTHOOK: query: lock database lockneg4 exclusive
POSTHOOK: type: LOCKDATABASE
+POSTHOOK: Input: database:lockneg4
+POSTHOOK: Output: database:lockneg4
PREHOOK: query: lock database lockneg4 shared
PREHOOK: type: LOCKDATABASE
+PREHOOK: Input: database:lockneg4
+PREHOOK: Output: database:lockneg4
Unable to acquire EXPLICIT, SHARED lock lockneg4 after 1 attempts.
FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask
http://git-wip-us.apache.org/repos/asf/hive/blob/6bfec2e9/ql/src/test/results/clientnegative/lockneg_try_drop_locked_db.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/lockneg_try_drop_locked_db.q.out b/ql/src/test/results/clientnegative/lockneg_try_drop_locked_db.q.out
index 5ac4e3c..e66965e 100644
--- a/ql/src/test/results/clientnegative/lockneg_try_drop_locked_db.q.out
+++ b/ql/src/test/results/clientnegative/lockneg_try_drop_locked_db.q.out
@@ -6,8 +6,12 @@ POSTHOOK: type: CREATEDATABASE
POSTHOOK: Output: database:lockneg9
PREHOOK: query: lock database lockneg9 shared
PREHOOK: type: LOCKDATABASE
+PREHOOK: Input: database:lockneg9
+PREHOOK: Output: database:lockneg9
POSTHOOK: query: lock database lockneg9 shared
POSTHOOK: type: LOCKDATABASE
+POSTHOOK: Input: database:lockneg9
+POSTHOOK: Output: database:lockneg9
PREHOOK: query: show locks database lockneg9
PREHOOK: type: SHOWLOCKS
POSTHOOK: query: show locks database lockneg9
http://git-wip-us.apache.org/repos/asf/hive/blob/6bfec2e9/ql/src/test/results/clientnegative/lockneg_try_lock_db_in_use.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/lockneg_try_lock_db_in_use.q.out b/ql/src/test/results/clientnegative/lockneg_try_lock_db_in_use.q.out
index 5486151..e5c8f3e 100644
--- a/ql/src/test/results/clientnegative/lockneg_try_lock_db_in_use.q.out
+++ b/ql/src/test/results/clientnegative/lockneg_try_lock_db_in_use.q.out
@@ -34,13 +34,19 @@ POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpar
POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
PREHOOK: query: lock database lockneg2 shared
PREHOOK: type: LOCKDATABASE
+PREHOOK: Input: database:lockneg2
+PREHOOK: Output: database:lockneg2
POSTHOOK: query: lock database lockneg2 shared
POSTHOOK: type: LOCKDATABASE
+POSTHOOK: Input: database:lockneg2
+POSTHOOK: Output: database:lockneg2
PREHOOK: query: show locks
PREHOOK: type: SHOWLOCKS
POSTHOOK: query: show locks
POSTHOOK: type: SHOWLOCKS
PREHOOK: query: lock database lockneg2 exclusive
PREHOOK: type: LOCKDATABASE
+PREHOOK: Input: database:lockneg2
+PREHOOK: Output: database:lockneg2
Unable to acquire EXPLICIT, EXCLUSIVE lock lockneg2 after 1 attempts.
FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask
[46/50] [abbrv] hive git commit: HIVE-13367: Extending HPLSQL parser
(Dmitry Tolpeko reviewed by Alan Gates)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/queries/db/create_procedure_no_params.sql
----------------------------------------------------------------------
diff --git a/hplsql/src/test/queries/db/create_procedure_no_params.sql b/hplsql/src/test/queries/db/create_procedure_no_params.sql
new file mode 100644
index 0000000..bbacb29
--- /dev/null
+++ b/hplsql/src/test/queries/db/create_procedure_no_params.sql
@@ -0,0 +1,25 @@
+CREATE OR REPLACE PROCEDURE show_the_date
+IS
+today DATE DEFAULT DATE '2016-03-10';
+BEGIN
+-- Display the date.
+DBMS_OUTPUT.PUT_LINE ('Today is ' || today);
+END show_the_date;
+
+CREATE OR REPLACE PROCEDURE show_the_date2()
+IS
+today DATE DEFAULT DATE '2016-03-10';
+BEGIN
+-- Display the date.
+DBMS_OUTPUT.PUT_LINE ('Today is ' || today);
+END show_the_date2;
+
+call show_the_date;
+call show_the_date2;
+
+DECLARE
+today DATE DEFAULT DATE '2016-03-10';
+BEGIN
+-- Display the date.
+DBMS_OUTPUT.PUT_LINE ('Today is ' || today);
+END;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/queries/db/describe.sql
----------------------------------------------------------------------
diff --git a/hplsql/src/test/queries/db/describe.sql b/hplsql/src/test/queries/db/describe.sql
new file mode 100644
index 0000000..f87b532
--- /dev/null
+++ b/hplsql/src/test/queries/db/describe.sql
@@ -0,0 +1,3 @@
+describe src;
+desc src;
+desc table src;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/queries/db/execute.sql
----------------------------------------------------------------------
diff --git a/hplsql/src/test/queries/db/execute.sql b/hplsql/src/test/queries/db/execute.sql
new file mode 100644
index 0000000..abc23c4
--- /dev/null
+++ b/hplsql/src/test/queries/db/execute.sql
@@ -0,0 +1,7 @@
+DECLARE cnt INT;
+EXECUTE 'SELECT COUNT(*) FROM src' INTO cnt;
+PRINT cnt;
+
+DECLARE sql STRING = 'SELECT COUNT(*) FROM src';
+EXECUTE sql INTO cnt;
+PRINT cnt;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/queries/db/expression.sql
----------------------------------------------------------------------
diff --git a/hplsql/src/test/queries/db/expression.sql b/hplsql/src/test/queries/db/expression.sql
new file mode 100644
index 0000000..cd82e7a
--- /dev/null
+++ b/hplsql/src/test/queries/db/expression.sql
@@ -0,0 +1 @@
+select 0.12*0.14, 0.13 / 0.11, 1 + 2, 1 - 2, (1-2)+3, 3 * (2-1) from src limit 1;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/queries/db/for.sql
----------------------------------------------------------------------
diff --git a/hplsql/src/test/queries/db/for.sql b/hplsql/src/test/queries/db/for.sql
new file mode 100644
index 0000000..19dac9a
--- /dev/null
+++ b/hplsql/src/test/queries/db/for.sql
@@ -0,0 +1 @@
+for item in (select code from sample_07 limit 10 ) loop print(cast(item.code as varchar2(100))+' aa') end loop;
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/queries/db/insert.sql
----------------------------------------------------------------------
diff --git a/hplsql/src/test/queries/db/insert.sql b/hplsql/src/test/queries/db/insert.sql
new file mode 100644
index 0000000..b67796a
--- /dev/null
+++ b/hplsql/src/test/queries/db/insert.sql
@@ -0,0 +1,3 @@
+insert overwrite table src_insert select value from src;
+
+insert into table src_insert select value from src;
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/queries/db/insert_directory.sql
----------------------------------------------------------------------
diff --git a/hplsql/src/test/queries/db/insert_directory.sql b/hplsql/src/test/queries/db/insert_directory.sql
new file mode 100644
index 0000000..b9a58d1
--- /dev/null
+++ b/hplsql/src/test/queries/db/insert_directory.sql
@@ -0,0 +1,12 @@
+insert overwrite directory /tmp/src1
+ select * from src;
+
+insert overwrite local directory /tmp/src2
+ select * from src;
+
+insert overwrite local directory '/tmp/src3'
+ 'select * from ' || 'src';
+
+declare tabname string = 'src';
+insert overwrite local directory '/tmp/src_' || date '2016-03-28'
+ 'select * from ' || tabname;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/queries/db/schema.sql
----------------------------------------------------------------------
diff --git a/hplsql/src/test/queries/db/schema.sql b/hplsql/src/test/queries/db/schema.sql
index 0c41569..e003934 100644
--- a/hplsql/src/test/queries/db/schema.sql
+++ b/hplsql/src/test/queries/db/schema.sql
@@ -29,4 +29,12 @@ select
cast(key as double)/10 c10,
date '2015-09-07' c11,
cast(date '2015-09-07' as timestamp) c12
-from src;
\ No newline at end of file
+from src;
+
+create table if not exists src_empty (
+ c1 string)
+;
+
+create table if not exists src_insert (
+ c1 string)
+;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/queries/db/truncate_table.sql
----------------------------------------------------------------------
diff --git a/hplsql/src/test/queries/db/truncate_table.sql b/hplsql/src/test/queries/db/truncate_table.sql
new file mode 100644
index 0000000..de2ac83
--- /dev/null
+++ b/hplsql/src/test/queries/db/truncate_table.sql
@@ -0,0 +1,2 @@
+truncate table src_empty;
+truncate src_empty;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/queries/local/create_function3.sql
----------------------------------------------------------------------
diff --git a/hplsql/src/test/queries/local/create_function3.sql b/hplsql/src/test/queries/local/create_function3.sql
new file mode 100644
index 0000000..840c095
--- /dev/null
+++ b/hplsql/src/test/queries/local/create_function3.sql
@@ -0,0 +1,58 @@
+FUNCTION gettype(tag1 varchar2, srcvalue varchar2) return varchar2 as
+ tmpVar varchar2(10);
+ BEGIN
+
+ if srcvalue is null or srcvalue = '@I' then
+ return '@I';
+ end if;
+
+ if trim(tag1) = 'WHMM' then
+ return '002';
+ end if;
+
+ if trim(tag1) = 'TCPJ' and srcvalue = '010105' then
+ return '010105';
+ end if;
+
+ if trim(tag1) = 'TCPJ' and srcvalue != '010105' then
+ return '003';
+ end if;
+
+ if trim(tag1) = 'TCPJ' and srcvalue != '010105' then
+ return '003_ticket';
+ end if;
+
+ if trim(tag1) = 'TCJY' and srcvalue != '010105' then
+ return '003_ticket';
+ end if;
+
+ if trim(tag1) = 'TCJY' and srcvalue != '010105' then
+ return '003_ticket';
+ end if;
+
+ if trim(tag1) = 'YHHPD' then
+ return '002_foreign';
+ end if;
+
+ if trim(tag1) = 'WHWZ' then
+ return '002_foreign';
+ end if;
+
+ if trim(tag1) = 'WHLZ' then
+ return '002_foreign';
+ end if;
+
+ if trim(tag1) = 'DEWZ' then
+ return '024_out';
+ end if;
+
+ if trim(tag1) = 'DELZ' then
+ return '024_out';
+ end if;
+
+ return srcvalue;
+
+ END;
+
+ gettype('YHHPD', 'a');
+ gettype('YHHPD', '@I');
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/queries/local/create_function4.sql
----------------------------------------------------------------------
diff --git a/hplsql/src/test/queries/local/create_function4.sql b/hplsql/src/test/queries/local/create_function4.sql
new file mode 100644
index 0000000..21986e0
--- /dev/null
+++ b/hplsql/src/test/queries/local/create_function4.sql
@@ -0,0 +1,19 @@
+FUNCTION get(CODE VARCHAR2) RETURN VARCHAR2 AS
+ TMPVAR VARCHAR2(10);
+ BEGIN
+
+ TMPVAR := '';
+
+ IF TRIM(TMPVAR) = '' THEN
+ RETURN '00080000';
+ ELSE
+ RETURN TRIM(TMPVAR);
+ END IF;
+ EXCEPTION
+ WHEN NO_DATA_FOUND THEN
+ RETURN '00080000';
+ WHEN OTHERS THEN
+ RAISE;
+ END;
+
+get('abc');
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/queries/local/create_procedure3.sql
----------------------------------------------------------------------
diff --git a/hplsql/src/test/queries/local/create_procedure3.sql b/hplsql/src/test/queries/local/create_procedure3.sql
new file mode 100644
index 0000000..b4c9fad
--- /dev/null
+++ b/hplsql/src/test/queries/local/create_procedure3.sql
@@ -0,0 +1,29 @@
+create or replace procedure proc1(out v_msg string)
+declare
+ num1 int=0;
+ num2 int=10;
+ num3 int=30;
+begin
+ print num1;
+ print num2;
+ print num3;
+ set v_msg = 'Completed';
+end;
+
+create or replace procedure proc2(out v_msg string)
+is
+ num1 int=0;
+ num2 int=10;
+ num3 int=30;
+begin
+ print num1;
+ print num2;
+ print num3;
+ set v_msg = 'Completed';
+end;
+
+declare str string;
+call proc1(str);
+print str;
+call proc2(str);
+print str;
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/queries/local/declare3.sql
----------------------------------------------------------------------
diff --git a/hplsql/src/test/queries/local/declare3.sql b/hplsql/src/test/queries/local/declare3.sql
new file mode 100644
index 0000000..09b94d6
--- /dev/null
+++ b/hplsql/src/test/queries/local/declare3.sql
@@ -0,0 +1,7 @@
+declare i2 int2 = 1;
+declare i4 int4 = 1;
+declare i8 int8 = 1;
+
+print i2+1;
+print i4+1;
+print i8+1;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/queries/local/if.sql
----------------------------------------------------------------------
diff --git a/hplsql/src/test/queries/local/if.sql b/hplsql/src/test/queries/local/if.sql
index 2de3045..4b3f161 100644
--- a/hplsql/src/test/queries/local/if.sql
+++ b/hplsql/src/test/queries/local/if.sql
@@ -4,9 +4,9 @@ DECLARE count INT;
SET state = 'CA';
SET count = 1;
-/*IF count = 1 THEN
+IF count = 1 THEN
PRINT 'True block - Correct';
-END IF;*/
+END IF;
IF state = 'CA' THEN
PRINT 'True block - Correct';
@@ -54,7 +54,7 @@ IF state = 'CA'
ELSE
PRINT 'False block - Incorrect';
-PRINT 'Transact-SQL - BEGIN-END block';
+PRINT 'Transact-SQL - BEGIN-END block';
IF state = 'CA'
BEGIN
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/queries/local/interval.sql
----------------------------------------------------------------------
diff --git a/hplsql/src/test/queries/local/interval.sql b/hplsql/src/test/queries/local/interval.sql
index 7962f2d..e40fcbd 100644
--- a/hplsql/src/test/queries/local/interval.sql
+++ b/hplsql/src/test/queries/local/interval.sql
@@ -12,4 +12,6 @@ TIMESTAMP '2015-03-12 10:10:10.000' - 1 MICROSECOND;
DATE '2015-03-12' - NVL(NULL, 3) DAYS;
TIMESTAMP '2015-03-12' - NVL(NULL, 3) DAYS;
-TIMESTAMP '2015-03-12' - 1 DAY - 1 MICROSECOND;
\ No newline at end of file
+TIMESTAMP '2015-03-12' - 1 DAY - 1 MICROSECOND;
+
+date '2016-01-27' - interval '3' day;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/queries/local/replace.sql
----------------------------------------------------------------------
diff --git a/hplsql/src/test/queries/local/replace.sql b/hplsql/src/test/queries/local/replace.sql
new file mode 100644
index 0000000..820aa06
--- /dev/null
+++ b/hplsql/src/test/queries/local/replace.sql
@@ -0,0 +1 @@
+replace('2016-03-03', '-', '');
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/queries/offline/create_table_mssql2.sql
----------------------------------------------------------------------
diff --git a/hplsql/src/test/queries/offline/create_table_mssql2.sql b/hplsql/src/test/queries/offline/create_table_mssql2.sql
new file mode 100644
index 0000000..d8a5805
--- /dev/null
+++ b/hplsql/src/test/queries/offline/create_table_mssql2.sql
@@ -0,0 +1,33 @@
+USE [mic.gr]
+GO
+
+/****** Object: Table [dbo].[downloads] Script Date: 03/11/2016 11:46:35 ******/
+SET ANSI_NULLS ON
+GO
+
+SET QUOTED_IDENTIFIER ON
+GO
+
+SET ANSI_PADDING ON
+GO
+
+CREATE TABLE [dbo].[downloads](
+ [id] [int] IDENTITY(1,1) NOT NULL,
+ [fileName] [char](255) NOT NULL,
+ [fileType] [char](10) NULL,
+ [downloads] [int] NULL,
+ [fromDate] [char](40) NULL,
+ [untilDate] [char](40) NULL,
+ CONSTRAINT [PK_downloads] PRIMARY KEY CLUSTERED
+(
+ [id] ASC
+)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
+) ON [PRIMARY]
+
+GO
+
+SET ANSI_PADDING OFF
+GO
+
+ALTER TABLE [dbo].[downloads] ADD CONSTRAINT [DF_downloads_downloads] DEFAULT (0) FOR [downloads]
+GO
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/queries/offline/create_table_mysql.sql
----------------------------------------------------------------------
diff --git a/hplsql/src/test/queries/offline/create_table_mysql.sql b/hplsql/src/test/queries/offline/create_table_mysql.sql
new file mode 100644
index 0000000..86afa28
--- /dev/null
+++ b/hplsql/src/test/queries/offline/create_table_mysql.sql
@@ -0,0 +1,5 @@
+CREATE TABLE IF NOT EXISTS `users` (
+ `id` int(11) NOT NULL AUTO_INCREMENT,
+ `name` text NOT NULL,
+ PRIMARY KEY (`id`)
+) ENGINE=MyISAM DEFAULT CHARSET=latin1 COMMENT='users table' AUTO_INCREMENT=20 ;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/queries/offline/create_table_ora2.sql
----------------------------------------------------------------------
diff --git a/hplsql/src/test/queries/offline/create_table_ora2.sql b/hplsql/src/test/queries/offline/create_table_ora2.sql
new file mode 100644
index 0000000..028c9de
--- /dev/null
+++ b/hplsql/src/test/queries/offline/create_table_ora2.sql
@@ -0,0 +1,6 @@
+CREATE TABLE "default"."AUDIT_LOGS"
+ ( "RUN_ID" VARCHAR2(36) NOT NULL ENABLE,
+ "FILE_NAME" VARCHAR2(255) NOT NULL ENABLE,
+ "RUN_DATE" DATE NOT NULL ENABLE,
+ CONSTRAINT "AUDIT_LOGS_PK" PRIMARY KEY ("RUN_ID") ENABLE
+ );
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/queries/offline/create_table_pg.sql
----------------------------------------------------------------------
diff --git a/hplsql/src/test/queries/offline/create_table_pg.sql b/hplsql/src/test/queries/offline/create_table_pg.sql
new file mode 100644
index 0000000..e0068a4
--- /dev/null
+++ b/hplsql/src/test/queries/offline/create_table_pg.sql
@@ -0,0 +1,5 @@
+create table i1 (
+ c1 int2,
+ c2 int4,
+ c3 int8
+)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/queries/offline/update.sql
----------------------------------------------------------------------
diff --git a/hplsql/src/test/queries/offline/update.sql b/hplsql/src/test/queries/offline/update.sql
new file mode 100644
index 0000000..f40fb0f
--- /dev/null
+++ b/hplsql/src/test/queries/offline/update.sql
@@ -0,0 +1,33 @@
+UPDATE tab T SET (c1) = TRIM(c1) WHERE T.c2 = 'A';
+
+UPDATE tab T
+ SET c1 = TRIM(c1)
+ WHERE T.c2 = 'A';
+
+UPDATE tab SET c1 = '0011' WHERE c1 = '0021';
+
+UPDATE tab T SET c1 = TRIM(c1), c3 = TRIM(c3) WHERE T.col2 = 'A';
+
+UPDATE tab T
+ SET (c1, c3) = (TRIM(c1), TRIM(c3))
+ WHERE T.col2 = 'A';
+
+UPDATE tab T
+ SET (c1, c2, c3, c4) =
+ (SELECT c1,
+ c2,
+ TRIM(c3),
+ c4
+ FROM tab2 C
+ WHERE C.c1 = T.c1)
+ WHERE T.c2 = 'A';
+
+UPDATE tab T
+ SET (c1) =
+ (SELECT c1 FROM tab2 C WHERE C.c1 = T.c1)
+ WHERE T.c2 = 'a';
+
+UPDATE tab T
+ SET c1 =
+ (SELECT c1 FROM tab2 C WHERE C.c1 = T.c1)
+ WHERE T.c2 = 'a';
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/results/db/create_drop_database.out.txt
----------------------------------------------------------------------
diff --git a/hplsql/src/test/results/db/create_drop_database.out.txt b/hplsql/src/test/results/db/create_drop_database.out.txt
new file mode 100644
index 0000000..194fbe6
--- /dev/null
+++ b/hplsql/src/test/results/db/create_drop_database.out.txt
@@ -0,0 +1,8 @@
+Ln:1 CREATE DATABASE
+Ln:1 create database test20160303
+Ln:2 CREATE DATABASE
+Ln:2 create database if not exists test1 comment 'abc' location '/users'
+Ln:4 DROP
+Ln:4 DROP DATABASE IF EXISTS test20160303
+Ln:5 DROP
+Ln:5 DROP DATABASE test1
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/results/db/create_procedure_mssql.out.txt
----------------------------------------------------------------------
diff --git a/hplsql/src/test/results/db/create_procedure_mssql.out.txt b/hplsql/src/test/results/db/create_procedure_mssql.out.txt
index 187e5bb..5214aa9 100644
--- a/hplsql/src/test/results/db/create_procedure_mssql.out.txt
+++ b/hplsql/src/test/results/db/create_procedure_mssql.out.txt
@@ -37,7 +37,7 @@ Ln:38 SELECT 'Correct' FROM src LIMIT 1
Ln:38 SELECT completed successfully
Ln:38 Standalone SELECT executed: 1 columns in the result set
Correct
-Ln:52 EXEC PROCEDURE spTest3
+Ln:51 EXEC PROCEDURE spTest3
Ln:44 SELECT
Ln:44 SELECT 'Correct' FROM src LIMIT 1
Ln:44 SELECT completed successfully
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/results/db/create_procedure_no_params.out.txt
----------------------------------------------------------------------
diff --git a/hplsql/src/test/results/db/create_procedure_no_params.out.txt b/hplsql/src/test/results/db/create_procedure_no_params.out.txt
new file mode 100644
index 0000000..005c4a6
--- /dev/null
+++ b/hplsql/src/test/results/db/create_procedure_no_params.out.txt
@@ -0,0 +1,10 @@
+Ln:1 CREATE PROCEDURE show_the_date
+Ln:9 CREATE PROCEDURE show_the_date2
+Ln:17 EXEC PROCEDURE show_the_date
+Ln:3 DECLARE today DATE = 2016-03-10
+Today is 2016-03-10
+Ln:18 EXEC PROCEDURE show_the_date2
+Ln:11 DECLARE today DATE = 2016-03-10
+Today is 2016-03-10
+Ln:21 DECLARE today DATE = 2016-03-10
+Today is 2016-03-10
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/results/db/create_procedure_return_cursor.out.txt
----------------------------------------------------------------------
diff --git a/hplsql/src/test/results/db/create_procedure_return_cursor.out.txt b/hplsql/src/test/results/db/create_procedure_return_cursor.out.txt
index 81aa6c8..a4370fe 100644
--- a/hplsql/src/test/results/db/create_procedure_return_cursor.out.txt
+++ b/hplsql/src/test/results/db/create_procedure_return_cursor.out.txt
@@ -2,7 +2,7 @@ Ln:1 CREATE PROCEDURE spResultSet1
Ln:9 CREATE PROCEDURE spResultSet2
Ln:20 DECLARE v1 VARCHAR
Ln:21 DECLARE v2 VARCHAR
-EXEC PROCEDURE spResultSet1
+Ln:23 EXEC PROCEDURE spResultSet1
Ln:4 DECLARE CURSOR cur1
Ln:6 OPEN
Ln:6 cur1: SELECT 'A', 'A1' FROM src LIMIT 3
@@ -32,7 +32,7 @@ A - A1
Ln:30 FETCH
Ln:27 WHILE - LEFT
Ln:32 CLOSE
-EXEC PROCEDURE spResultSet2
+Ln:34 EXEC PROCEDURE spResultSet2
Ln:12 DECLARE CURSOR cur1
Ln:14 DECLARE CURSOR cur2
Ln:16 OPEN
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/results/db/create_procedure_return_cursor2.out.txt
----------------------------------------------------------------------
diff --git a/hplsql/src/test/results/db/create_procedure_return_cursor2.out.txt b/hplsql/src/test/results/db/create_procedure_return_cursor2.out.txt
index 40f2c33..853de2d 100644
--- a/hplsql/src/test/results/db/create_procedure_return_cursor2.out.txt
+++ b/hplsql/src/test/results/db/create_procedure_return_cursor2.out.txt
@@ -4,7 +4,7 @@ Ln:20 DECLARE v1 VARCHAR
Ln:21 DECLARE v2 VARCHAR
Ln:22 DECLARE loc1 RESULT_SET_LOCATOR VARYING
Ln:23 DECLARE loc2 RESULT_SET_LOCATOR VARYING
-EXEC PROCEDURE spResultSet1
+Ln:25 EXEC PROCEDURE spResultSet1
Ln:4 DECLARE CURSOR cur1
Ln:6 OPEN
Ln:6 cur1: SELECT 'A', 'A1' FROM src LIMIT 3
@@ -35,7 +35,7 @@ A - A1
Ln:34 FETCH
Ln:31 WHILE - LEFT
Ln:36 CLOSE
-EXEC PROCEDURE spResultSet2
+Ln:38 EXEC PROCEDURE spResultSet2
Ln:12 DECLARE CURSOR cur1
Ln:14 DECLARE CURSOR cur2
Ln:16 OPEN
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/results/db/describe.out.txt
----------------------------------------------------------------------
diff --git a/hplsql/src/test/results/db/describe.out.txt b/hplsql/src/test/results/db/describe.out.txt
new file mode 100644
index 0000000..9772a1b
--- /dev/null
+++ b/hplsql/src/test/results/db/describe.out.txt
@@ -0,0 +1,12 @@
+Ln:1 DESCRIBE
+Ln:1 DESCRIBE src
+key string default
+value string default
+Ln:2 DESCRIBE
+Ln:2 DESCRIBE src
+key string default
+value string default
+Ln:3 DESCRIBE
+Ln:3 DESCRIBE src
+key string default
+value string default
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/results/db/execute.out.txt
----------------------------------------------------------------------
diff --git a/hplsql/src/test/results/db/execute.out.txt b/hplsql/src/test/results/db/execute.out.txt
new file mode 100644
index 0000000..24fa646
--- /dev/null
+++ b/hplsql/src/test/results/db/execute.out.txt
@@ -0,0 +1,14 @@
+Ln:1 DECLARE cnt INT
+Ln:2 EXECUTE
+Ln:2 SQL statement: SELECT COUNT(*) FROM src
+Ln:2 COLUMN: _c0, bigint
+Ln:2 SET cnt = 500
+Ln:3 PRINT
+500
+Ln:5 DECLARE sql STRING = 'SELECT COUNT(*) FROM src'
+Ln:6 EXECUTE
+Ln:6 SQL statement: SELECT COUNT(*) FROM src
+Ln:6 COLUMN: _c0, bigint
+Ln:6 SET cnt = 500
+Ln:7 PRINT
+500
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/results/db/expression.out.txt
----------------------------------------------------------------------
diff --git a/hplsql/src/test/results/db/expression.out.txt b/hplsql/src/test/results/db/expression.out.txt
new file mode 100644
index 0000000..2535464
--- /dev/null
+++ b/hplsql/src/test/results/db/expression.out.txt
@@ -0,0 +1,5 @@
+Ln:1 SELECT
+Ln:1 select 0.12 * 0.14, 0.13 / 0.11, 1 + 2, 1 - 2, (1 - 2) + 3, 3 * (2 - 1) from src LIMIT 1
+Ln:1 SELECT completed successfully
+Ln:1 Standalone SELECT executed: 6 columns in the result set
+0.016800000000000002 1.1818181818181819 3 -1 2 3
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/results/db/for.out.txt
----------------------------------------------------------------------
diff --git a/hplsql/src/test/results/db/for.out.txt b/hplsql/src/test/results/db/for.out.txt
new file mode 100644
index 0000000..e6730ba
--- /dev/null
+++ b/hplsql/src/test/results/db/for.out.txt
@@ -0,0 +1,44 @@
+Ln:1 FOR CURSOR - ENTERED
+Ln:1 select code from sample_07 LIMIT 10
+Ln:1 SELECT completed successfully
+Ln:1 COLUMN: code, string
+Ln:1 SET code = 00-0000
+Ln:1 PRINT
+00-0000 aa
+Ln:1 COLUMN: code, string
+Ln:1 SET code = 11-0000
+Ln:1 PRINT
+11-0000 aa
+Ln:1 COLUMN: code, string
+Ln:1 SET code = 11-1011
+Ln:1 PRINT
+11-1011 aa
+Ln:1 COLUMN: code, string
+Ln:1 SET code = 11-1021
+Ln:1 PRINT
+11-1021 aa
+Ln:1 COLUMN: code, string
+Ln:1 SET code = 11-1031
+Ln:1 PRINT
+11-1031 aa
+Ln:1 COLUMN: code, string
+Ln:1 SET code = 11-2011
+Ln:1 PRINT
+11-2011 aa
+Ln:1 COLUMN: code, string
+Ln:1 SET code = 11-2021
+Ln:1 PRINT
+11-2021 aa
+Ln:1 COLUMN: code, string
+Ln:1 SET code = 11-2022
+Ln:1 PRINT
+11-2022 aa
+Ln:1 COLUMN: code, string
+Ln:1 SET code = 11-2031
+Ln:1 PRINT
+11-2031 aa
+Ln:1 COLUMN: code, string
+Ln:1 SET code = 11-3011
+Ln:1 PRINT
+11-3011 aa
+Ln:1 FOR CURSOR - LEFT
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/results/db/insert.out.txt
----------------------------------------------------------------------
diff --git a/hplsql/src/test/results/db/insert.out.txt b/hplsql/src/test/results/db/insert.out.txt
new file mode 100644
index 0000000..e0362ae
--- /dev/null
+++ b/hplsql/src/test/results/db/insert.out.txt
@@ -0,0 +1,4 @@
+Ln:1 INSERT SELECT
+Ln:1 insert overwrite table src_insert select value from src
+Ln:3 INSERT SELECT
+Ln:3 insert into table src_insert select value from src
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/results/db/insert_directory.out.txt
----------------------------------------------------------------------
diff --git a/hplsql/src/test/results/db/insert_directory.out.txt b/hplsql/src/test/results/db/insert_directory.out.txt
new file mode 100644
index 0000000..73f0b07
--- /dev/null
+++ b/hplsql/src/test/results/db/insert_directory.out.txt
@@ -0,0 +1,9 @@
+Ln:1 INSERT DIRECTORY
+Ln:1 insert overwrite directory '/tmp/src1' select * from src
+Ln:4 INSERT DIRECTORY
+Ln:4 insert overwrite local directory '/tmp/src2' select * from src
+Ln:7 INSERT DIRECTORY
+Ln:7 insert overwrite local directory '/tmp/src3' select * from src
+Ln:10 DECLARE tabname string = 'src'
+Ln:11 INSERT DIRECTORY
+Ln:11 insert overwrite local directory '/tmp/src_2016-03-28' select * from src
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/results/db/rowtype_attribute.out.txt
----------------------------------------------------------------------
diff --git a/hplsql/src/test/results/db/rowtype_attribute.out.txt b/hplsql/src/test/results/db/rowtype_attribute.out.txt
index fc22370..e846d88 100644
--- a/hplsql/src/test/results/db/rowtype_attribute.out.txt
+++ b/hplsql/src/test/results/db/rowtype_attribute.out.txt
@@ -30,7 +30,7 @@ Ln:15 SET key = A
Ln:15 COLUMN: value, string
Ln:15 SET value = B
Ln:17 PRINT
-null
+AB
Ln:15 FOR CURSOR - LEFT
Ln:20 EXECUTE
Ln:20 SQL statement: SELECT 'A' AS key, 'B' AS value FROM src LIMIT 1
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/results/db/truncate_table.out.txt
----------------------------------------------------------------------
diff --git a/hplsql/src/test/results/db/truncate_table.out.txt b/hplsql/src/test/results/db/truncate_table.out.txt
new file mode 100644
index 0000000..fa46916
--- /dev/null
+++ b/hplsql/src/test/results/db/truncate_table.out.txt
@@ -0,0 +1,4 @@
+Ln:1 TRUNCATE
+Ln:1 TRUNCATE TABLE src_empty
+Ln:2 TRUNCATE
+Ln:2 TRUNCATE TABLE src_empty
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/results/local/create_function3.out.txt
----------------------------------------------------------------------
diff --git a/hplsql/src/test/results/local/create_function3.out.txt b/hplsql/src/test/results/local/create_function3.out.txt
new file mode 100644
index 0000000..90974ed
--- /dev/null
+++ b/hplsql/src/test/results/local/create_function3.out.txt
@@ -0,0 +1,22 @@
+Ln:1 CREATE FUNCTION gettype
+Ln:57 EXEC FUNCTION gettype
+Ln:57 SET PARAM tag1 = YHHPD
+Ln:57 SET PARAM srcvalue = a
+Ln:5 IF
+Ln:9 IF
+Ln:13 IF
+Ln:17 IF
+Ln:21 IF
+Ln:25 IF
+Ln:29 IF
+Ln:33 IF
+Ln:33 IF TRUE executed
+Ln:34 RETURN
+002_foreign
+Ln:58 EXEC FUNCTION gettype
+Ln:58 SET PARAM tag1 = YHHPD
+Ln:58 SET PARAM srcvalue = @I
+Ln:5 IF
+Ln:5 IF TRUE executed
+Ln:6 RETURN
+@I
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/results/local/create_function4.out.txt
----------------------------------------------------------------------
diff --git a/hplsql/src/test/results/local/create_function4.out.txt b/hplsql/src/test/results/local/create_function4.out.txt
new file mode 100644
index 0000000..d8daae5
--- /dev/null
+++ b/hplsql/src/test/results/local/create_function4.out.txt
@@ -0,0 +1,9 @@
+Ln:1 CREATE FUNCTION get
+Ln:19 EXEC FUNCTION get
+Ln:19 SET PARAM CODE = abc
+Ln:5 SET TMPVAR = ''
+Ln:7 IF
+Ln:7 IF TRUE executed
+Ln:8 RETURN
+Ln:15 EXCEPTION HANDLER
+00080000
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/results/local/create_package.out.txt
----------------------------------------------------------------------
diff --git a/hplsql/src/test/results/local/create_package.out.txt b/hplsql/src/test/results/local/create_package.out.txt
index ee7a9a0..4b631e5 100644
--- a/hplsql/src/test/results/local/create_package.out.txt
+++ b/hplsql/src/test/results/local/create_package.out.txt
@@ -24,7 +24,7 @@ Ln:20 SET PARAM p1 = 3
a: 3
b: 1
p1: 3
-Ln:21 EXEC PROCEDURE sp2
+EXEC PROCEDURE sp2
Ln:21 SET PARAM p2 = 1
pack1.a: 3
p2: 1
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/results/local/create_procedure3.out.txt
----------------------------------------------------------------------
diff --git a/hplsql/src/test/results/local/create_procedure3.out.txt b/hplsql/src/test/results/local/create_procedure3.out.txt
new file mode 100644
index 0000000..c2a7c79
--- /dev/null
+++ b/hplsql/src/test/results/local/create_procedure3.out.txt
@@ -0,0 +1,31 @@
+Ln:1 CREATE PROCEDURE proc1
+Ln:13 CREATE PROCEDURE proc2
+Ln:25 DECLARE str string
+Ln:26 EXEC PROCEDURE proc1
+Ln:26 SET PARAM v_msg = null
+Ln:3 DECLARE num1 int = 0
+Ln:4 DECLARE num2 int = 10
+Ln:5 DECLARE num3 int = 30
+Ln:7 PRINT
+0
+Ln:8 PRINT
+10
+Ln:9 PRINT
+30
+Ln:10 SET v_msg = 'Completed'
+Ln:27 PRINT
+Completed
+Ln:28 EXEC PROCEDURE proc2
+Ln:15 DECLARE num1 int = 0
+Ln:16 DECLARE num2 int = 10
+Ln:17 DECLARE num3 int = 30
+Ln:28 SET PARAM v_msg = Completed
+Ln:19 PRINT
+0
+Ln:20 PRINT
+10
+Ln:21 PRINT
+30
+Ln:22 SET v_msg = 'Completed'
+Ln:29 PRINT
+Completed
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/results/local/create_procedure_no_params.out.txt
----------------------------------------------------------------------
diff --git a/hplsql/src/test/results/local/create_procedure_no_params.out.txt b/hplsql/src/test/results/local/create_procedure_no_params.out.txt
index 9e3bde9..9767e35 100644
--- a/hplsql/src/test/results/local/create_procedure_no_params.out.txt
+++ b/hplsql/src/test/results/local/create_procedure_no_params.out.txt
@@ -1,24 +1,24 @@
Ln:1 CREATE PROCEDURE sp1
Ln:6 CREATE PROCEDURE sp2
-EXEC PROCEDURE sp1
+Ln:11 EXEC PROCEDURE sp1
Ln:3 PRINT
a
-EXEC PROCEDURE sp1
+Ln:12 EXEC PROCEDURE sp1
Ln:3 PRINT
a
-EXEC PROCEDURE sp1
+Ln:13 EXEC PROCEDURE sp1
Ln:3 PRINT
a
EXEC PROCEDURE sp1
Ln:3 PRINT
a
-EXEC PROCEDURE sp2
+Ln:16 EXEC PROCEDURE sp2
Ln:8 PRINT
b
-EXEC PROCEDURE sp2
+Ln:17 EXEC PROCEDURE sp2
Ln:8 PRINT
b
-EXEC PROCEDURE sp2
+Ln:18 EXEC PROCEDURE sp2
Ln:8 PRINT
b
EXEC PROCEDURE sp2
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/results/local/declare3.out.txt
----------------------------------------------------------------------
diff --git a/hplsql/src/test/results/local/declare3.out.txt b/hplsql/src/test/results/local/declare3.out.txt
new file mode 100644
index 0000000..f568aaa
--- /dev/null
+++ b/hplsql/src/test/results/local/declare3.out.txt
@@ -0,0 +1,9 @@
+Ln:1 DECLARE i2 int2 = 1
+Ln:2 DECLARE i4 int4 = 1
+Ln:3 DECLARE i8 int8 = 1
+Ln:5 PRINT
+2
+Ln:6 PRINT
+2
+Ln:7 PRINT
+2
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/results/local/if.out.txt
----------------------------------------------------------------------
diff --git a/hplsql/src/test/results/local/if.out.txt b/hplsql/src/test/results/local/if.out.txt
index 1da8142..b34c1fc 100644
--- a/hplsql/src/test/results/local/if.out.txt
+++ b/hplsql/src/test/results/local/if.out.txt
@@ -2,6 +2,10 @@ Ln:1 DECLARE state VARCHAR
Ln:2 DECLARE count INT
Ln:4 SET state = 'CA'
Ln:5 SET count = 1
+Ln:7 IF
+Ln:7 IF TRUE executed
+Ln:8 PRINT
+True block - Correct
Ln:11 IF
Ln:11 IF TRUE executed
Ln:12 PRINT
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/results/local/interval.out.txt
----------------------------------------------------------------------
diff --git a/hplsql/src/test/results/local/interval.out.txt b/hplsql/src/test/results/local/interval.out.txt
index 2dcdcd5..9420877 100644
--- a/hplsql/src/test/results/local/interval.out.txt
+++ b/hplsql/src/test/results/local/interval.out.txt
@@ -9,3 +9,4 @@
2015-03-09
2015-03-09 00:00:00
2015-03-10 23:59:59
+2016-01-24
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/results/local/replace.out.txt
----------------------------------------------------------------------
diff --git a/hplsql/src/test/results/local/replace.out.txt b/hplsql/src/test/results/local/replace.out.txt
new file mode 100644
index 0000000..7c8bd69
--- /dev/null
+++ b/hplsql/src/test/results/local/replace.out.txt
@@ -0,0 +1 @@
+20160303
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/results/offline/create_table_mssql2.out.txt
----------------------------------------------------------------------
diff --git a/hplsql/src/test/results/offline/create_table_mssql2.out.txt b/hplsql/src/test/results/offline/create_table_mssql2.out.txt
new file mode 100644
index 0000000..a765c4a
--- /dev/null
+++ b/hplsql/src/test/results/offline/create_table_mssql2.out.txt
@@ -0,0 +1,10 @@
+Ln:1 USE
+Ln:1 SQL statement: USE `mic.gr`
+Ln:14 CREATE TABLE
+Ln:14 CREATE TABLE `downloads` (`id` int,
+`fileName` char(255),
+`fileType` char(10),
+`downloads` int,
+`fromDate` char(40),
+`untilDate` char(40)
+)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/results/offline/create_table_mysql.out.txt
----------------------------------------------------------------------
diff --git a/hplsql/src/test/results/offline/create_table_mysql.out.txt b/hplsql/src/test/results/offline/create_table_mysql.out.txt
new file mode 100644
index 0000000..b835135
--- /dev/null
+++ b/hplsql/src/test/results/offline/create_table_mysql.out.txt
@@ -0,0 +1,4 @@
+Ln:1 CREATE TABLE
+Ln:1 CREATE TABLE `users` (`id` int,
+`name` STRING
+) COMMENT 'users table'
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/results/offline/create_table_ora2.out.txt
----------------------------------------------------------------------
diff --git a/hplsql/src/test/results/offline/create_table_ora2.out.txt b/hplsql/src/test/results/offline/create_table_ora2.out.txt
new file mode 100644
index 0000000..5d4e107
--- /dev/null
+++ b/hplsql/src/test/results/offline/create_table_ora2.out.txt
@@ -0,0 +1,5 @@
+Ln:1 CREATE TABLE
+Ln:1 CREATE TABLE `default`.`AUDIT_LOGS` (`RUN_ID` STRING,
+`FILE_NAME` STRING,
+`RUN_DATE` DATE
+)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/results/offline/create_table_pg.out.txt
----------------------------------------------------------------------
diff --git a/hplsql/src/test/results/offline/create_table_pg.out.txt b/hplsql/src/test/results/offline/create_table_pg.out.txt
new file mode 100644
index 0000000..cad5488
--- /dev/null
+++ b/hplsql/src/test/results/offline/create_table_pg.out.txt
@@ -0,0 +1,5 @@
+Ln:1 CREATE TABLE
+Ln:1 create table i1 (c1 SMALLINT,
+c2 INT,
+c3 BIGINT
+)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/results/offline/update.out.txt
----------------------------------------------------------------------
diff --git a/hplsql/src/test/results/offline/update.out.txt b/hplsql/src/test/results/offline/update.out.txt
new file mode 100644
index 0000000..fadaac0
--- /dev/null
+++ b/hplsql/src/test/results/offline/update.out.txt
@@ -0,0 +1,34 @@
+Ln:1 UPDATE
+Ln:1 UPDATE tab T SET (c1) = TRIM(c1) WHERE T.c2 = 'A'
+Ln:3 UPDATE
+Ln:3 UPDATE tab T
+ SET c1 = TRIM(c1)
+ WHERE T.c2 = 'A'
+Ln:7 UPDATE
+Ln:7 UPDATE tab SET c1 = '0011' WHERE c1 = '0021'
+Ln:9 UPDATE
+Ln:9 UPDATE tab T SET c1 = TRIM(c1), c3 = TRIM(c3) WHERE T.col2 = 'A'
+Ln:11 UPDATE
+Ln:11 UPDATE tab T
+ SET (c1, c3) = (TRIM(c1), TRIM(c3))
+ WHERE T.col2 = 'A'
+Ln:15 UPDATE
+Ln:15 UPDATE tab T
+ SET (c1, c2, c3, c4) =
+ (SELECT c1,
+ c2,
+ TRIM(c3),
+ c4
+ FROM tab2 C
+ WHERE C.c1 = T.c1)
+ WHERE T.c2 = 'A'
+Ln:25 UPDATE
+Ln:25 UPDATE tab T
+ SET (c1) =
+ (SELECT c1 FROM tab2 C WHERE C.c1 = T.c1)
+ WHERE T.c2 = 'a'
+Ln:30 UPDATE
+Ln:30 UPDATE tab T
+ SET c1 =
+ (SELECT c1 FROM tab2 C WHERE C.c1 = T.c1)
+ WHERE T.c2 = 'a'
\ No newline at end of file
[47/50] [abbrv] hive git commit: HIVE-13367: Extending HPLSQL parser
(Dmitry Tolpeko reviewed by Alan Gates)
Posted by jd...@apache.org.
HIVE-13367: Extending HPLSQL parser (Dmitry Tolpeko reviewed by Alan Gates)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/39d66a43
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/39d66a43
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/39d66a43
Branch: refs/heads/llap
Commit: 39d66a439c02ea2b5c7501b362c0d8f9b8d22cc0
Parents: 55383d8
Author: Dmitry Tolpeko <dm...@gmail.com>
Authored: Wed Mar 30 00:19:11 2016 -0700
Committer: Dmitry Tolpeko <dm...@gmail.com>
Committed: Wed Mar 30 00:19:11 2016 -0700
----------------------------------------------------------------------
.../antlr4/org/apache/hive/hplsql/Hplsql.g4 | 266 +++++++++---
.../main/java/org/apache/hive/hplsql/Conf.java | 2 +-
.../main/java/org/apache/hive/hplsql/Conn.java | 3 +-
.../java/org/apache/hive/hplsql/Converter.java | 15 +-
.../main/java/org/apache/hive/hplsql/Exec.java | 106 ++++-
.../java/org/apache/hive/hplsql/Expression.java | 13 +
.../main/java/org/apache/hive/hplsql/File.java | 11 +
.../main/java/org/apache/hive/hplsql/Ftp.java | 415 +++++++++++++++++++
.../main/java/org/apache/hive/hplsql/Meta.java | 35 +-
.../java/org/apache/hive/hplsql/Package.java | 3 +
.../main/java/org/apache/hive/hplsql/Row.java | 4 +-
.../java/org/apache/hive/hplsql/Select.java | 6 +-
.../main/java/org/apache/hive/hplsql/Stmt.java | 167 +++++++-
.../main/java/org/apache/hive/hplsql/Utils.java | 40 ++
.../main/java/org/apache/hive/hplsql/Var.java | 18 +-
.../apache/hive/hplsql/functions/Function.java | 40 +-
.../hive/hplsql/functions/FunctionDatetime.java | 40 ++
.../hive/hplsql/functions/FunctionString.java | 26 +-
.../org/apache/hive/hplsql/TestHplsqlLocal.java | 28 +-
.../apache/hive/hplsql/TestHplsqlOffline.java | 25 ++
.../test/queries/db/create_drop_database.sql | 5 +
.../queries/db/create_procedure_no_params.sql | 25 ++
hplsql/src/test/queries/db/describe.sql | 3 +
hplsql/src/test/queries/db/execute.sql | 7 +
hplsql/src/test/queries/db/expression.sql | 1 +
hplsql/src/test/queries/db/for.sql | 1 +
hplsql/src/test/queries/db/insert.sql | 3 +
hplsql/src/test/queries/db/insert_directory.sql | 12 +
hplsql/src/test/queries/db/schema.sql | 10 +-
hplsql/src/test/queries/db/truncate_table.sql | 2 +
.../src/test/queries/local/create_function3.sql | 58 +++
.../src/test/queries/local/create_function4.sql | 19 +
.../test/queries/local/create_procedure3.sql | 29 ++
hplsql/src/test/queries/local/declare3.sql | 7 +
hplsql/src/test/queries/local/if.sql | 6 +-
hplsql/src/test/queries/local/interval.sql | 4 +-
hplsql/src/test/queries/local/replace.sql | 1 +
.../queries/offline/create_table_mssql2.sql | 33 ++
.../test/queries/offline/create_table_mysql.sql | 5 +
.../test/queries/offline/create_table_ora2.sql | 6 +
.../test/queries/offline/create_table_pg.sql | 5 +
hplsql/src/test/queries/offline/update.sql | 33 ++
.../results/db/create_drop_database.out.txt | 8 +
.../results/db/create_procedure_mssql.out.txt | 2 +-
.../db/create_procedure_no_params.out.txt | 10 +
.../db/create_procedure_return_cursor.out.txt | 4 +-
.../db/create_procedure_return_cursor2.out.txt | 4 +-
hplsql/src/test/results/db/describe.out.txt | 12 +
hplsql/src/test/results/db/execute.out.txt | 14 +
hplsql/src/test/results/db/expression.out.txt | 5 +
hplsql/src/test/results/db/for.out.txt | 44 ++
hplsql/src/test/results/db/insert.out.txt | 4 +
.../test/results/db/insert_directory.out.txt | 9 +
.../test/results/db/rowtype_attribute.out.txt | 2 +-
.../src/test/results/db/truncate_table.out.txt | 4 +
.../test/results/local/create_function3.out.txt | 22 +
.../test/results/local/create_function4.out.txt | 9 +
.../test/results/local/create_package.out.txt | 2 +-
.../results/local/create_procedure3.out.txt | 31 ++
.../local/create_procedure_no_params.out.txt | 12 +-
hplsql/src/test/results/local/declare3.out.txt | 9 +
hplsql/src/test/results/local/if.out.txt | 4 +
hplsql/src/test/results/local/interval.out.txt | 1 +
hplsql/src/test/results/local/replace.out.txt | 1 +
.../results/offline/create_table_mssql2.out.txt | 10 +
.../results/offline/create_table_mysql.out.txt | 4 +
.../results/offline/create_table_ora2.out.txt | 5 +
.../results/offline/create_table_pg.out.txt | 5 +
hplsql/src/test/results/offline/update.out.txt | 34 ++
69 files changed, 1672 insertions(+), 137 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/main/antlr4/org/apache/hive/hplsql/Hplsql.g4
----------------------------------------------------------------------
diff --git a/hplsql/src/main/antlr4/org/apache/hive/hplsql/Hplsql.g4 b/hplsql/src/main/antlr4/org/apache/hive/hplsql/Hplsql.g4
index a1083ee..b84116f 100644
--- a/hplsql/src/main/antlr4/org/apache/hive/hplsql/Hplsql.g4
+++ b/hplsql/src/main/antlr4/org/apache/hive/hplsql/Hplsql.g4
@@ -23,30 +23,38 @@ program : block ;
block : ((begin_end_block | stmt) T_GO?)+ ; // Multiple consecutive blocks/statements
begin_end_block :
- declare_block? T_BEGIN block exception_block? T_END
+ declare_block? T_BEGIN block exception_block? block_end
;
single_block_stmt : // Single BEGIN END block (but nested blocks are possible) or single statement
- T_BEGIN block T_END
+ T_BEGIN block exception_block? block_end
| stmt T_SEMICOLON?
;
+
+block_end :
+ {!_input.LT(2).getText().equalsIgnoreCase("TRANSACTION")}? T_END
+ ;
proc_block :
- T_BEGIN block T_END
+ begin_end_block
| stmt+ T_GO?
;
stmt :
assignment_stmt
| allocate_cursor_stmt
+ | alter_table_stmt
| associate_locator_stmt
+ | begin_transaction_stmt
| break_stmt
| call_stmt
| close_stmt
| cmp_stmt
+ | copy_from_ftp_stmt
| copy_from_local_stmt
| copy_stmt
| commit_stmt
+ | create_database_stmt
| create_function_stmt
| create_index_stmt
| create_local_temp_table_stmt
@@ -56,7 +64,9 @@ stmt :
| create_table_stmt
| declare_stmt
| delete_stmt
+ | describe_stmt
| drop_stmt
+ | end_transaction_stmt
| exec_stmt
| exit_stmt
| fetch_stmt
@@ -65,6 +75,7 @@ stmt :
| if_stmt
| include_stmt
| insert_stmt
+ | insert_directory_stmt
| get_diag_stmt
| grant_stmt
| leave_stmt
@@ -72,6 +83,7 @@ stmt :
| merge_stmt
| open_stmt
| print_stmt
+ | raise_stmt
| resignal_stmt
| return_stmt
| rollback_stmt
@@ -79,6 +91,7 @@ stmt :
| signal_stmt
| update_stmt
| use_stmt
+ | truncate_stmt
| values_into_stmt
| while_stmt
| label
@@ -123,6 +136,7 @@ assignment_stmt_item :
assignment_stmt_single_item :
ident T_COLON? T_EQUAL expr
+ | T_OPEN_P ident T_CLOSE_P T_COLON? T_EQUAL expr
;
assignment_stmt_multiple_item :
@@ -141,6 +155,10 @@ associate_locator_stmt :
T_ASSOCIATE (T_RESULT T_SET)? (T_LOCATOR | T_LOCATORS) T_OPEN_P ident (T_COMMA ident)* T_CLOSE_P T_WITH T_PROCEDURE ident
;
+begin_transaction_stmt :
+ T_BEGIN T_TRANSACTION
+ ;
+
break_stmt :
T_BREAK
;
@@ -156,7 +174,10 @@ declare_stmt : // Declaration statement
declare_block : // Declaration block
T_DECLARE declare_stmt_item T_SEMICOLON (declare_stmt_item T_SEMICOLON)*
;
-
+
+declare_block_inplace :
+ declare_stmt_item T_SEMICOLON (declare_stmt_item T_SEMICOLON)*
+ ;
declare_stmt_item :
declare_cursor_item
@@ -200,7 +221,7 @@ create_table_stmt :
;
create_local_temp_table_stmt :
- T_CREATE (T_LOCAL T_TEMPORARY | (T_SET | T_MULTISET)? T_VOLATILE) T_TABLE ident T_OPEN_P create_table_columns T_CLOSE_P create_table_options?
+ T_CREATE (T_LOCAL T_TEMPORARY | (T_SET | T_MULTISET)? T_VOLATILE) T_TABLE ident create_table_preoptions? T_OPEN_P create_table_columns T_CLOSE_P create_table_options?
;
create_table_columns :
@@ -223,10 +244,12 @@ create_table_column_inline_cons :
| T_UNIQUE
| T_REFERENCES table_name T_OPEN_P ident T_CLOSE_P create_table_fk_action*
| T_IDENTITY T_OPEN_P L_INT (T_COMMA L_INT)* T_CLOSE_P
+ | T_AUTO_INCREMENT
+ | T_ENABLE
;
create_table_column_cons :
- T_PRIMARY T_KEY T_CLUSTERED? T_OPEN_P ident (T_ASC | T_DESC)? (T_COMMA ident (T_ASC | T_DESC)?)* T_CLOSE_P index_storage_clause?
+ T_PRIMARY T_KEY T_CLUSTERED? T_OPEN_P ident (T_ASC | T_DESC)? (T_COMMA ident (T_ASC | T_DESC)?)* T_CLOSE_P T_ENABLE? index_storage_clause?
| T_FOREIGN T_KEY T_OPEN_P ident (T_COMMA ident)* T_CLOSE_P T_REFERENCES table_name T_OPEN_P ident (T_COMMA ident)* T_CLOSE_P create_table_fk_action*
;
@@ -234,6 +257,14 @@ create_table_fk_action :
T_ON (T_UPDATE | T_DELETE) (T_NO T_ACTION | T_RESTRICT | T_SET T_NULL | T_SET T_DEFAULT | T_CASCADE)
;
+create_table_preoptions :
+ (T_COMMA create_table_preoptions_item)+
+ ;
+
+create_table_preoptions_item :
+ T_NO? T_LOG
+ ;
+
create_table_options :
create_table_options_item+
;
@@ -243,7 +274,8 @@ create_table_options_item :
| create_table_options_ora_item
| create_table_options_db2_item
| create_table_options_hive_item
- | create_table_options_mssql_item
+ | create_table_options_mssql_item
+ | create_table_options_mysql_item
;
create_table_options_ora_item :
@@ -284,6 +316,31 @@ create_table_options_mssql_item :
T_ON ident
| T_TEXTIMAGE_ON ident
;
+
+create_table_options_mysql_item :
+ T_AUTO_INCREMENT T_EQUAL? expr
+ | T_COMMENT T_EQUAL? expr
+ | T_DEFAULT? (T_CHARACTER T_SET | T_CHARSET) T_EQUAL? expr
+ | T_ENGINE T_EQUAL? expr
+ ;
+
+alter_table_stmt :
+ T_ALTER T_TABLE table_name alter_table_item
+ ;
+
+alter_table_item :
+ alter_table_add_constraint
+ ;
+
+alter_table_add_constraint :
+ T_ADD2 (T_CONSTRAINT ident)? alter_table_add_constraint_item
+ ;
+
+alter_table_add_constraint_item :
+ T_PRIMARY T_KEY T_CLUSTERED? T_OPEN_P ident (T_ASC | T_DESC)? (T_COMMA ident (T_ASC | T_DESC)?)* T_CLOSE_P T_ENABLE? index_storage_clause?
+ | T_FOREIGN T_KEY T_OPEN_P ident (T_COMMA ident)* T_CLOSE_P T_REFERENCES table_name T_OPEN_P ident (T_COMMA ident)* T_CLOSE_P create_table_fk_action*
+ | T_DEFAULT expr T_FOR ident
+ ;
dtype : // Data types
T_CHAR
@@ -299,6 +356,9 @@ dtype : // Data types
| T_DOUBLE T_PRECISION?
| T_FLOAT
| T_INT
+ | T_INT2
+ | T_INT4
+ | T_INT8
| T_INTEGER
| T_NCHAR
| T_NVARCHAR
@@ -336,9 +396,18 @@ dtype_default : // Default clause in variable declaration
T_COLON? T_EQUAL expr
| T_DEFAULT expr
;
+
+create_database_stmt :
+ T_CREATE (T_DATABASE | T_SCHEMA) (T_IF T_NOT T_EXISTS)? expr create_database_option*
+ ;
+
+create_database_option :
+ T_COMMENT expr
+ | T_LOCATION expr
+ ;
create_function_stmt :
- (T_ALTER | T_CREATE (T_OR T_REPLACE)? | T_REPLACE)? T_FUNCTION ident create_routine_params? create_function_return (T_AS | T_IS)? single_block_stmt
+ (T_ALTER | T_CREATE (T_OR T_REPLACE)? | T_REPLACE)? T_FUNCTION ident create_routine_params? create_function_return (T_AS | T_IS)? declare_block_inplace? single_block_stmt
;
create_function_return :
@@ -374,12 +443,15 @@ package_body_item :
;
create_procedure_stmt :
- (T_ALTER | T_CREATE (T_OR T_REPLACE)? | T_REPLACE)? (T_PROCEDURE | T_PROC) ident create_routine_params? create_routine_options? (T_AS | T_IS)? label? proc_block (ident T_SEMICOLON)?
+ (T_ALTER | T_CREATE (T_OR T_REPLACE)? | T_REPLACE)? (T_PROCEDURE | T_PROC) ident create_routine_params? create_routine_options? (T_AS | T_IS)? declare_block_inplace? label? proc_block (ident T_SEMICOLON)?
;
create_routine_params :
T_OPEN_P T_CLOSE_P
- | T_OPEN_P? create_routine_param_item (T_COMMA create_routine_param_item)* T_CLOSE_P?
+ | T_OPEN_P create_routine_param_item (T_COMMA create_routine_param_item)* T_CLOSE_P
+ | {!_input.LT(1).getText().equalsIgnoreCase("IS") &&
+ !_input.LT(1).getText().equalsIgnoreCase("AS")}?
+ create_routine_param_item (T_COMMA create_routine_param_item)*
;
create_routine_param_item :
@@ -398,6 +470,11 @@ create_routine_option :
drop_stmt : // DROP statement
T_DROP T_TABLE (T_IF T_EXISTS)? table_name
+ | T_DROP (T_DATABASE | T_SCHEMA) (T_IF T_EXISTS)? expr
+ ;
+
+end_transaction_stmt :
+ T_END T_TRANSACTION
;
exec_stmt : // EXEC, EXECUTE IMMEDIATE statement
@@ -444,6 +521,10 @@ insert_stmt_rows :
insert_stmt_row:
T_OPEN_P expr (T_COMMA expr)* T_CLOSE_P
;
+
+insert_directory_stmt :
+ T_INSERT T_OVERWRITE T_LOCAL? T_DIRECTORY expr_file expr_select
+ ;
exit_stmt :
T_EXIT L_ID? (T_WHEN bool_expr)?
@@ -502,6 +583,10 @@ cmp_source :
(table_name where_clause? | T_OPEN_P select_stmt T_CLOSE_P) (T_AT ident)?
;
+copy_from_ftp_stmt :
+ T_COPY T_FROM T_FTP expr copy_ftp_option*
+ ;
+
copy_from_local_stmt : // COPY FROM LOCAL statement
T_COPY T_FROM T_LOCAL copy_source (T_COMMA copy_source)* T_TO copy_target copy_file_option*
;
@@ -531,6 +616,18 @@ copy_file_option :
| T_OVERWRITE
;
+copy_ftp_option :
+ T_USER expr
+ | T_PWD expr
+ | T_DIR (file_name | expr)
+ | T_FILES expr
+ | T_NEW
+ | T_OVERWRITE
+ | T_SUBDIR
+ | T_SESSIONS expr
+ | T_TO T_LOCAL? (file_name | expr)
+ ;
+
commit_stmt : // COMMIT statement
T_COMMIT T_WORK?
;
@@ -556,6 +653,10 @@ print_stmt : // PRINT statement
| T_PRINT T_OPEN_P expr T_CLOSE_P
;
+raise_stmt :
+ T_RAISE
+ ;
+
resignal_stmt : // RESIGNAL statement
T_RESIGNAL (T_SQLSTATE T_VALUE? expr (T_SET T_MESSAGE_TEXT T_EQUAL expr)? )?
;
@@ -571,6 +672,7 @@ rollback_stmt : // ROLLBACK statement
set_session_option :
set_current_schema_option
| set_mssql_session_option
+ | set_teradata_session_option
;
set_current_schema_option :
@@ -586,9 +688,17 @@ set_mssql_session_option :
(T_ON | T_OFF)
;
+set_teradata_session_option :
+ T_QUERY_BAND T_EQUAL (expr | T_NONE) T_UPDATE? T_FOR (T_TRANSACTION | T_SESSION)
+ ;
+
signal_stmt : // SIGNAL statement
T_SIGNAL ident
;
+
+truncate_stmt :
+ T_TRUNCATE T_TABLE? table_name
+ ;
use_stmt : // USE statement
T_USE expr
@@ -708,7 +818,7 @@ from_join_clause :
;
from_join_type_clause :
- T_INNER T_JOIN
+ T_INNER? T_JOIN
| (T_LEFT | T_RIGHT | T_FULL) T_OUTER? T_JOIN
;
@@ -761,7 +871,11 @@ select_options_item :
;
update_stmt : // UPDATE statement
- T_UPDATE update_table T_SET assignment_stmt_item (T_COMMA assignment_stmt_item)* where_clause? update_upsert?
+ T_UPDATE update_table T_SET update_assignment where_clause? update_upsert?
+ ;
+
+update_assignment :
+ assignment_stmt_item (T_COMMA assignment_stmt_item)*
;
update_table :
@@ -787,13 +901,17 @@ merge_condition :
merge_action :
T_INSERT insert_stmt_cols? T_VALUES insert_stmt_row
- | T_UPDATE T_SET assignment_stmt_item (T_COMMA assignment_stmt_item)*
+ | T_UPDATE T_SET assignment_stmt_item (T_COMMA assignment_stmt_item)* where_clause?
| T_DELETE
;
delete_stmt : // DELETE statement
T_DELETE T_FROM? table_name (T_AS? ident)? where_clause?
;
+
+describe_stmt :
+ (T_DESCRIBE | T_DESC) T_TABLE? table_name
+ ;
bool_expr : // Boolean condition
T_NOT? T_OPEN_P bool_expr T_CLOSE_P
@@ -843,13 +961,14 @@ bool_expr_binary_operator :
| T_NOT? (T_LIKE | T_RLIKE | T_REGEXP)
;
-expr :
+expr :
expr interval_item
| expr T_MUL expr
| expr T_DIV expr
| expr T_ADD expr
| expr T_SUB expr
| T_OPEN_P expr T_CLOSE_P
+ | expr_interval
| expr_concat
| expr_case
| expr_cursor_attribute
@@ -859,7 +978,6 @@ expr :
| expr_atom
;
-
expr_atom :
date_literal
| timestamp_literal
@@ -867,11 +985,13 @@ expr_atom :
| ident
| string
| dec_number
- | interval_number
| int_number
| null_const
;
+expr_interval :
+ T_INTERVAL expr interval_item
+ ;
interval_item :
T_DAY
| T_DAYS
@@ -879,10 +999,6 @@ interval_item :
| T_MICROSECONDS
;
-interval_number :
- int_number interval_item
- ;
-
expr_concat : // String concatenation operator
expr_concat_item (T_PIPE | T_CONCAT) expr_concat_item ((T_PIPE | T_CONCAT) expr_concat_item)*
;
@@ -943,7 +1059,7 @@ expr_func_over_clause :
;
expr_func_partition_by_clause :
- T_PARTITION T_BY ident (T_COMMA ident)*
+ T_PARTITION T_BY expr (T_COMMA expr)*
;
expr_spec_func :
@@ -978,18 +1094,25 @@ expr_func_params :
func_param :
{!_input.LT(1).getText().equalsIgnoreCase("INTO")}? (ident T_EQUAL T_GREATER?)? expr
;
+
+expr_select :
+ select_stmt
+ | expr
+ ;
+expr_file :
+ file_name
+ | expr
+ ;
+
hive :
T_HIVE hive_item*
;
hive_item :
- P_e expr
- | P_f expr
- | P_hiveconf L_ID T_EQUAL expr
- | P_i expr
- | P_S
- | P_h
+ T_SUB ident expr
+ | T_SUB ident L_ID T_EQUAL expr
+ | T_SUB ident
;
host :
@@ -1006,7 +1129,7 @@ host_stmt :
;
file_name :
- L_FILE | '/'? ident ('/' ident)*
+ L_FILE | ('/' | '.' '/')? ident ('/' ident)*
;
date_literal : // DATE 'YYYY-MM-DD' literal
@@ -1047,6 +1170,7 @@ null_const : // NULL constant
non_reserved_words : // Tokens that are not reserved words and can be used as identifiers
T_ACTION
| T_ACTIVITY_COUNT
+ | T_ADD2
| T_ALL
| T_ALLOCATE
| T_ALTER
@@ -1057,6 +1181,7 @@ non_reserved_words : // Tokens that are not reserved words
| T_ASC
| T_ASSOCIATE
| T_AT
+ | T_AUTO_INCREMENT
| T_AVG
| T_BATCHSIZE
| T_BEGIN
@@ -1076,12 +1201,14 @@ non_reserved_words : // Tokens that are not reserved words
| T_CASESPECIFIC
| T_CAST
| T_CHAR
- | T_CHARACTER
+ | T_CHARACTER
+ | T_CHARSET
| T_CLIENT
| T_CLOSE
| T_CLUSTERED
| T_CMP
- | T_COLLECTION
+ | T_COLLECTION
+ | T_COMMENT
| T_CONSTANT
| T_COPY
| T_COMMIT
@@ -1102,6 +1229,7 @@ non_reserved_words : // Tokens that are not reserved words
| T_CURRENT_TIMESTAMP
| T_CURRENT_USER
| T_CURSOR
+ | T_DATABASE
| T_DATE
| T_DATETIME
| T_DAY
@@ -1118,8 +1246,11 @@ non_reserved_words : // Tokens that are not reserved words
| T_DELIMITED
| T_DELIMITER
| T_DENSE_RANK
- | T_DESC
+ | T_DESC
+ | T_DESCRIBE
| T_DIAGNOSTICS
+ | T_DIR
+ | T_DIRECTORY
| T_DISTINCT
| T_DISTRIBUTE
| T_DO
@@ -1129,7 +1260,9 @@ non_reserved_words : // Tokens that are not reserved words
// T_ELSE reserved word
// T_ELSEIF reserved word
// T_ELSIF reserved word
- // T_END reserved word
+ // T_END reserved word
+ | T_ENABLE
+ | T_ENGINE
| T_ESCAPED
| T_EXCEPT
| T_EXEC
@@ -1141,14 +1274,16 @@ non_reserved_words : // Tokens that are not reserved words
| T_FALSE
| T_FETCH
| T_FIELDS
- | T_FILE
+ | T_FILE
+ | T_FILES
| T_FIRST_VALUE
| T_FLOAT
| T_FOR
| T_FOREIGN
| T_FORMAT
| T_FOUND
- | T_FROM
+ | T_FROM
+ | T_FTP
| T_FULL
| T_FUNCTION
| T_GET
@@ -1174,8 +1309,12 @@ non_reserved_words : // Tokens that are not reserved words
| T_INOUT
| T_INSERT
| T_INT
+ | T_INT2
+ | T_INT4
+ | T_INT8
| T_INTEGER
- | T_INTERSECT
+ | T_INTERSECT
+ | T_INTERVAL
| T_INTO
| T_INVOKER
| T_ITEMS
@@ -1194,10 +1333,12 @@ non_reserved_words : // Tokens that are not reserved words
| T_LIKE
| T_LIMIT
| T_LINES
- | T_LOCAL
+ | T_LOCAL
+ | T_LOCATION
| T_LOCATOR
| T_LOCATORS
| T_LOCKS
+ | T_LOG
| T_LOGGED
| T_LOGGING
| T_LOOP
@@ -1212,11 +1353,13 @@ non_reserved_words : // Tokens that are not reserved words
| T_MIN
| T_MULTISET
| T_NCHAR
+ | T_NEW
| T_NVARCHAR
| T_NO
| T_NOCOMPRESS
| T_NOCOUNT
| T_NOLOGGING
+ | T_NONE
| T_NOT
| T_NOTFOUND
// T_NULL reserved word
@@ -1245,8 +1388,11 @@ non_reserved_words : // Tokens that are not reserved words
| T_PRIMARY
| T_PRINT
| T_PROC
- | T_PROCEDURE
+ | T_PROCEDURE
+ | T_PWD
+ | T_QUERY_BAND
| T_QUOTED_IDENTIFIER
+ | T_RAISE
| T_RANK
| T_REAL
| T_REFERENCES
@@ -1273,7 +1419,9 @@ non_reserved_words : // Tokens that are not reserved words
| T_SECURITY
| T_SEGMENT
| T_SEL
- | T_SELECT
+ | T_SELECT
+ | T_SESSION
+ | T_SESSIONS
| T_SET
| T_SETS
| T_SHARE
@@ -1290,7 +1438,8 @@ non_reserved_words : // Tokens that are not reserved words
| T_STEP
| T_STDEV
| T_STORAGE
- | T_STRING
+ | T_STRING
+ | T_SUBDIR
| T_SUBSTRING
| T_SUM
| T_SYSDATE
@@ -1305,8 +1454,10 @@ non_reserved_words : // Tokens that are not reserved words
| T_TITLE
| T_TO
| T_TOP
+ | T_TRANSACTION
| T_TRIM
| T_TRUE
+ | T_TRUNCATE
// T_UNION reserved word
| T_UNIQUE
| T_UPDATE
@@ -1334,6 +1485,7 @@ non_reserved_words : // Tokens that are not reserved words
// Lexer rules
T_ACTION : A C T I O N ;
+T_ADD2 : A D D ;
T_ALL : A L L ;
T_ALLOCATE : A L L O C A T E ;
T_ALTER : A L T E R ;
@@ -1344,6 +1496,7 @@ T_AS : A S ;
T_ASC : A S C ;
T_ASSOCIATE : A S S O C I A T E ;
T_AT : A T ;
+T_AUTO_INCREMENT : A U T O '_' I N C R E M E N T ;
T_AVG : A V G ;
T_BATCHSIZE : B A T C H S I Z E ;
T_BEGIN : B E G I N ;
@@ -1365,18 +1518,20 @@ T_CASESPECIFIC : C A S E S P E C I F I C ;
T_CAST : C A S T ;
T_CHAR : C H A R ;
T_CHARACTER : C H A R A C T E R ;
+T_CHARSET : C H A R S E T ;
T_CLIENT : C L I E N T ;
T_CLOSE : C L O S E ;
T_CLUSTERED : C L U S T E R E D;
T_CMP : C M P ;
T_COLLECTION : C O L L E C T I O N ;
+T_COMMENT : C O M M E N T;
T_CONSTANT : C O N S T A N T ;
-T_COPY : C O P Y ;
T_COMMIT : C O M M I T ;
T_CONCAT : C O N C A T;
T_CONDITION : C O N D I T I O N ;
T_CONSTRAINT : C O N S T R A I N T ;
T_CONTINUE : C O N T I N U E ;
+T_COPY : C O P Y ;
T_COUNT : C O U N T ;
T_COUNT_BIG : C O U N T '_' B I G;
T_CREATE : C R E A T E ;
@@ -1386,6 +1541,7 @@ T_CS : C S;
T_CURRENT : C U R R E N T ;
T_CURRENT_SCHEMA : C U R R E N T '_' S C H E M A ;
T_CURSOR : C U R S O R ;
+T_DATABASE : D A T A B A S E;
T_DATE : D A T E ;
T_DATETIME : D A T E T I M E ;
T_DAY : D A Y ;
@@ -1402,7 +1558,10 @@ T_DELETE : D E L E T E ;
T_DELIMITED : D E L I M I T E D ;
T_DELIMITER : D E L I M I T E R ;
T_DESC : D E S C ;
+T_DESCRIBE : D E S C R I B E ;
T_DIAGNOSTICS : D I A G N O S T I C S ;
+T_DIR : D I R ;
+T_DIRECTORY : D I R E C T O R Y ;
T_DISTINCT : D I S T I N C T ;
T_DISTRIBUTE : D I S T R I B U T E ;
T_DO : D O ;
@@ -1412,7 +1571,9 @@ T_DYNAMIC : D Y N A M I C ;
T_ELSE : E L S E ;
T_ELSEIF : E L S E I F ;
T_ELSIF : E L S I F ;
+T_ENABLE : E N A B L E ;
T_END : E N D ;
+T_ENGINE : E N G I N E ;
T_ESCAPED : E S C A P E D ;
T_EXCEPT : E X C E P T ;
T_EXEC : E X E C ;
@@ -1425,12 +1586,14 @@ T_FALSE : F A L S E ;
T_FETCH : F E T C H ;
T_FIELDS : F I E L D S ;
T_FILE : F I L E ;
+T_FILES : F I L E S ;
T_FLOAT : F L O A T ;
T_FOR : F O R ;
T_FOREIGN : F O R E I G N ;
T_FORMAT : F O R M A T ;
T_FOUND : F O U N D ;
T_FROM : F R O M ;
+T_FTP : F T P ;
T_FULL : F U L L ;
T_FUNCTION : F U N C T I O N ;
T_GET : G E T ;
@@ -1456,8 +1619,12 @@ T_INNER : I N N E R ;
T_INOUT : I N O U T;
T_INSERT : I N S E R T ;
T_INT : I N T ;
+T_INT2 : I N T '2';
+T_INT4 : I N T '4';
+T_INT8 : I N T '8';
T_INTEGER : I N T E G E R ;
T_INTERSECT : I N T E R S E C T ;
+T_INTERVAL : I N T E R V A L ;
T_INTO : I N T O ;
T_INVOKER : I N V O K E R ;
T_IS : I S ;
@@ -1474,9 +1641,11 @@ T_LIKE : L I K E ;
T_LIMIT : L I M I T ;
T_LINES : L I N E S ;
T_LOCAL : L O C A L ;
+T_LOCATION : L O C A T I O N ;
T_LOCATOR : L O C A T O R ;
T_LOCATORS : L O C A T O R S ;
T_LOCKS : L O C K S ;
+T_LOG : L O G ;
T_LOGGED : L O G G E D ;
T_LOGGING : L O G G I N G ;
T_LOOP : L O O P ;
@@ -1491,11 +1660,13 @@ T_MICROSECONDS : M I C R O S E C O N D S;
T_MIN : M I N ;
T_MULTISET : M U L T I S E T ;
T_NCHAR : N C H A R ;
+T_NEW : N E W ;
T_NVARCHAR : N V A R C H A R ;
T_NO : N O ;
T_NOCOUNT : N O C O U N T ;
T_NOCOMPRESS : N O C O M P R E S S ;
T_NOLOGGING : N O L O G G I N G ;
+T_NONE : N O N E ;
T_NOT : N O T ;
T_NOTFOUND : N O T F O U N D ;
T_NULL : N U L L ;
@@ -1524,7 +1695,9 @@ T_PRIMARY : P R I M A R Y ;
T_PRINT : P R I N T ;
T_PROC : P R O C ;
T_PROCEDURE : P R O C E D U R E;
+T_QUERY_BAND : Q U E R Y '_' B A N D ;
T_QUOTED_IDENTIFIER : Q U O T E D '_' I D E N T I F I E R ;
+T_RAISE : R A I S E ;
T_REAL : R E A L ;
T_REFERENCES : R E F E R E N C E S ;
T_REGEXP : R E G E X P ;
@@ -1546,6 +1719,7 @@ T_ROWTYPE : R O W T Y P E ;
T_ROW_COUNT : R O W '_' C O U N T ;
T_RR : R R;
T_RS : R S ;
+T_PWD : P W D ;
T_TRIM : T R I M ;
T_SCHEMA : S C H E M A ;
T_SECURITY : S E C U R I T Y ;
@@ -1553,6 +1727,8 @@ T_SEGMENT : S E G M E N T ;
T_SEL : S E L ;
T_SELECT : S E L E C T ;
T_SET : S E T ;
+T_SESSION : S E S S I O N ;
+T_SESSIONS : S E S S I O N S ;
T_SETS : S E T S;
T_SHARE : S H A R E ;
T_SIGNAL : S I G N A L ;
@@ -1569,6 +1745,7 @@ T_SQLWARNING : S Q L W A R N I N G ;
T_STEP : S T E P ;
T_STORAGE : S T O R A G E ;
T_STRING : S T R I N G ;
+T_SUBDIR : S U B D I R ;
T_SUBSTRING : S U B S T R I N G ;
T_SUM : S U M ;
T_SYS_REFCURSOR : S Y S '_' R E F C U R S O R ;
@@ -1583,7 +1760,9 @@ T_TINYINT : T I N Y I N T ;
T_TITLE : T I T L E ;
T_TO : T O ;
T_TOP : T O P ;
+T_TRANSACTION : T R A N S A C T I O N ;
T_TRUE : T R U E ;
+T_TRUNCATE : T R U N C A T E;
T_TYPE : T Y P E ;
T_UNION : U N I O N ;
T_UNIQUE : U N I Q U E ;
@@ -1657,13 +1836,6 @@ T_CLOSE_SB : ']' ;
T_SEMICOLON : ';' ;
T_SUB : '-' ;
-P_e : '-e' ;
-P_f : '-f' ;
-P_hiveconf : '-hiveconf' ;
-P_i : '-i' ;
-P_S : '-S' ;
-P_h : '-h' ;
-
L_ID : L_ID_PART (L_BLANK* '.' L_BLANK* L_ID_PART)* // Identifier
;
L_S_STRING : '\'' (('\'' '\'') | ('\\' '\'') | ~('\''))* '\'' // Single quoted string literal
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/main/java/org/apache/hive/hplsql/Conf.java
----------------------------------------------------------------------
diff --git a/hplsql/src/main/java/org/apache/hive/hplsql/Conf.java b/hplsql/src/main/java/org/apache/hive/hplsql/Conf.java
index 88afbb5..c78dda4 100644
--- a/hplsql/src/main/java/org/apache/hive/hplsql/Conf.java
+++ b/hplsql/src/main/java/org/apache/hive/hplsql/Conf.java
@@ -57,7 +57,7 @@ public class Conf extends Configuration {
InsertValues insertValues = InsertValues.NATIVE;
TempTables tempTables = TempTables.NATIVE;
- String dualTable = "default.dual";
+ String dualTable = null;
String tempTablesSchema = "";
String tempTablesLocation = "/tmp/hplsql";
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/main/java/org/apache/hive/hplsql/Conn.java
----------------------------------------------------------------------
diff --git a/hplsql/src/main/java/org/apache/hive/hplsql/Conn.java b/hplsql/src/main/java/org/apache/hive/hplsql/Conn.java
index a0e0958..8599536 100644
--- a/hplsql/src/main/java/org/apache/hive/hplsql/Conn.java
+++ b/hplsql/src/main/java/org/apache/hive/hplsql/Conn.java
@@ -110,6 +110,7 @@ public class Conn {
if (stmt.execute(sql)) {
rs = stmt.getResultSet();
}
+ timer.stop();
query.set(conn, stmt, rs);
if (info) {
exec.info(null, "SQL statement executed successfully (" + timer.format() + ")");
@@ -205,7 +206,7 @@ public class Conn {
}
Class.forName(driver);
timer.start();
- Connection conn = DriverManager.getConnection(url.toString(), usr, pwd);
+ Connection conn = DriverManager.getConnection(url.toString().trim(), usr, pwd);
timer.stop();
if (info) {
exec.info(null, "Open connection: " + url + " (" + timer.format() + ")");
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/main/java/org/apache/hive/hplsql/Converter.java
----------------------------------------------------------------------
diff --git a/hplsql/src/main/java/org/apache/hive/hplsql/Converter.java b/hplsql/src/main/java/org/apache/hive/hplsql/Converter.java
index 9d8c77f..e19bcfc 100644
--- a/hplsql/src/main/java/org/apache/hive/hplsql/Converter.java
+++ b/hplsql/src/main/java/org/apache/hive/hplsql/Converter.java
@@ -44,13 +44,26 @@ public class Converter {
if (t.equalsIgnoreCase("BIT")) {
t = "TINYINT";
}
+ else if (t.equalsIgnoreCase("INT") || t.equalsIgnoreCase("INTEGER")) {
+ // MySQL can use INT(n)
+ }
+ else if (t.equalsIgnoreCase("INT2")) {
+ t = "SMALLINT";
+ }
+ else if (t.equalsIgnoreCase("INT4")) {
+ t = "INT";
+ }
+ else if (t.equalsIgnoreCase("INT8")) {
+ t = "BIGINT";
+ }
else if (t.equalsIgnoreCase("DATETIME") || t.equalsIgnoreCase("SMALLDATETIME")) {
t = "TIMESTAMP";
}
else if ((t.equalsIgnoreCase("VARCHAR") || t.equalsIgnoreCase("NVARCHAR")) && len.T_MAX() != null) {
t = "STRING";
}
- else if (t.equalsIgnoreCase("VARCHAR2") || t.equalsIgnoreCase("NCHAR") || t.equalsIgnoreCase("NVARCHAR")) {
+ else if (t.equalsIgnoreCase("VARCHAR2") || t.equalsIgnoreCase("NCHAR") || t.equalsIgnoreCase("NVARCHAR") ||
+ t.equalsIgnoreCase("TEXT")) {
t = "STRING";
}
else if (t.equalsIgnoreCase("NUMBER") || t.equalsIgnoreCase("NUMERIC")) {
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/main/java/org/apache/hive/hplsql/Exec.java
----------------------------------------------------------------------
diff --git a/hplsql/src/main/java/org/apache/hive/hplsql/Exec.java b/hplsql/src/main/java/org/apache/hive/hplsql/Exec.java
index 84193da..02605a8 100644
--- a/hplsql/src/main/java/org/apache/hive/hplsql/Exec.java
+++ b/hplsql/src/main/java/org/apache/hive/hplsql/Exec.java
@@ -50,7 +50,7 @@ import org.apache.hive.hplsql.functions.*;
*/
public class Exec extends HplsqlBaseVisitor<Integer> {
- public static final String VERSION = "HPL/SQL 0.3.13";
+ public static final String VERSION = "HPL/SQL 0.3.17";
public static final String SQLCODE = "SQLCODE";
public static final String SQLSTATE = "SQLSTATE";
public static final String HOSTCODE = "HOSTCODE";
@@ -783,11 +783,9 @@ public class Exec extends HplsqlBaseVisitor<Integer> {
new FunctionMisc(this).register(function);
new FunctionString(this).register(function);
new FunctionOra(this).register(function);
-
addVariable(new Var(SQLCODE, Var.Type.BIGINT, 0L));
addVariable(new Var(SQLSTATE, Var.Type.STRING, "00000"));
addVariable(new Var(HOSTCODE, Var.Type.BIGINT, 0L));
-
for (Map.Entry<String, String> v : arguments.getVars().entrySet()) {
addVariable(new Var(v.getKey(), Var.Type.STRING, v.getValue()));
}
@@ -826,7 +824,7 @@ public class Exec extends HplsqlBaseVisitor<Integer> {
}
execString = arguments.getExecString();
execFile = arguments.getFileName();
- execMain = arguments.getMain();
+ execMain = arguments.getMain();
if (arguments.hasTraceOption()) {
trace = true;
}
@@ -1066,7 +1064,7 @@ public class Exec extends HplsqlBaseVisitor<Integer> {
public Integer visitTable_name(HplsqlParser.Table_nameContext ctx) {
String name = ctx.getText();
String nameUp = name.toUpperCase();
- String nameNorm = meta.normalizeIdentifier(name);
+ String nameNorm = meta.normalizeObjectIdentifier(name);
String actualName = exec.managedTables.get(nameUp);
String conn = exec.objectConnMap.get(nameUp);
if (conn == null) {
@@ -1093,6 +1091,14 @@ public class Exec extends HplsqlBaseVisitor<Integer> {
public Integer visitInsert_stmt(HplsqlParser.Insert_stmtContext ctx) {
return exec.stmt.insert(ctx);
}
+
+ /**
+ * INSERT DIRECTORY statement
+ */
+ @Override
+ public Integer visitInsert_directory_stmt(HplsqlParser.Insert_directory_stmtContext ctx) {
+ return exec.stmt.insertDirectory(ctx);
+ }
/**
* EXCEPTION block
@@ -1214,6 +1220,14 @@ public class Exec extends HplsqlBaseVisitor<Integer> {
}
/**
+ * DESCRIBE statement
+ */
+ @Override
+ public Integer visitDescribe_stmt(HplsqlParser.Describe_stmtContext ctx) {
+ return exec.stmt.describe(ctx);
+ }
+
+ /**
* DROP statement
*/
@Override
@@ -1262,6 +1276,14 @@ public class Exec extends HplsqlBaseVisitor<Integer> {
}
/**
+ * COPY FROM FTP statement
+ */
+ @Override
+ public Integer visitCopy_from_ftp_stmt(HplsqlParser.Copy_from_ftp_stmtContext ctx) {
+ return new Ftp(exec).run(ctx);
+ }
+
+ /**
* COPY FROM LOCAL statement
*/
@Override
@@ -1331,6 +1353,11 @@ public class Exec extends HplsqlBaseVisitor<Integer> {
return 0;
}
+ @Override
+ public Integer visitCreate_table_options_mysql_item(HplsqlParser.Create_table_options_mysql_itemContext ctx) {
+ return exec.stmt.createTableMysqlOptions(ctx);
+ }
+
/**
* CREATE LOCAL TEMPORARY | VOLATILE TABLE statement
*/
@@ -1340,6 +1367,22 @@ public class Exec extends HplsqlBaseVisitor<Integer> {
}
/**
+ * ALTER TABLE statement
+ */
+ @Override
+ public Integer visitAlter_table_stmt(HplsqlParser.Alter_table_stmtContext ctx) {
+ return 0;
+ }
+
+ /**
+ * CREATE DATABASE | SCHEMA statement
+ */
+ @Override
+ public Integer visitCreate_database_stmt(HplsqlParser.Create_database_stmtContext ctx) {
+ return exec.stmt.createDatabase(ctx);
+ }
+
+ /**
* CREATE FUNCTION statement
*/
@Override
@@ -1520,6 +1563,34 @@ public class Exec extends HplsqlBaseVisitor<Integer> {
}
return 0;
}
+
+ /**
+ * Static SELECT statement (i.e. unquoted) or expression
+ */
+ @Override
+ public Integer visitExpr_select(HplsqlParser.Expr_selectContext ctx) {
+ if (ctx.select_stmt() != null) {
+ stackPush(new Var(evalPop(ctx.select_stmt())));
+ }
+ else {
+ visit(ctx.expr());
+ }
+ return 0;
+ }
+
+ /**
+ * File path (unquoted) or expression
+ */
+ @Override
+ public Integer visitExpr_file(HplsqlParser.Expr_fileContext ctx) {
+ if (ctx.file_name() != null) {
+ stackPush(new Var(ctx.file_name().getText()));
+ }
+ else {
+ visit(ctx.expr());
+ }
+ return 0;
+ }
/**
* Cursor attribute %ISOPEN, %FOUND and %NOTFOUND
@@ -1677,7 +1748,7 @@ public class Exec extends HplsqlBaseVisitor<Integer> {
executed = packCallContext.execProc(name, ctx.expr_func_params(), false /*trace error if not exists*/);
}
if (!executed) {
- exec.function.execProc(name, ctx.expr_func_params());
+ exec.function.execProc(name, ctx.expr_func_params(), ctx);
}
exec.inCallStmt = false;
return 0;
@@ -1748,6 +1819,14 @@ public class Exec extends HplsqlBaseVisitor<Integer> {
}
/**
+ * TRUNCATE statement
+ */
+ @Override
+ public Integer visitTruncate_stmt(HplsqlParser.Truncate_stmtContext ctx) {
+ return exec.stmt.truncate(ctx);
+ }
+
+ /**
* MAP OBJECT statement
*/
@Override
@@ -1845,15 +1924,16 @@ public class Exec extends HplsqlBaseVisitor<Integer> {
public Integer visitHive_item(HplsqlParser.Hive_itemContext ctx) {
Var params = stackPeek();
ArrayList<String> a = (ArrayList<String>)params.value;
- if(ctx.P_e() != null) {
+ String param = ctx.getChild(1).getText();
+ if (param.equals("e")) {
a.add("-e");
a.add(evalPop(ctx.expr()).toString());
}
- else if(ctx.P_f() != null) {
+ else if (param.equals("f")) {
a.add("-f");
a.add(evalPop(ctx.expr()).toString());
}
- else if(ctx.P_hiveconf() != null) {
+ else if (param.equals("hiveconf")) {
a.add("-hiveconf");
a.add(ctx.L_ID().toString() + "=" + evalPop(ctx.expr()).toString());
}
@@ -2007,7 +2087,7 @@ public class Exec extends HplsqlBaseVisitor<Integer> {
}
}
else {
- if (!exec.buildSql && !exec.inCallStmt && exec.function.isProc(ident) && exec.function.execProc(ident, null)) {
+ if (!exec.buildSql && !exec.inCallStmt && exec.function.isProc(ident) && exec.function.execProc(ident, null, ctx)) {
return 0;
}
else {
@@ -2041,11 +2121,11 @@ public class Exec extends HplsqlBaseVisitor<Integer> {
}
/**
- * Interval number (1 DAYS i.e)
+ * Interval expression (INTERVAL '1' DAY i.e)
*/
@Override
- public Integer visitInterval_number(HplsqlParser.Interval_numberContext ctx) {
- int num = evalPop(ctx.int_number()).intValue();
+ public Integer visitExpr_interval(HplsqlParser.Expr_intervalContext ctx) {
+ int num = evalPop(ctx.expr()).intValue();
Interval interval = new Interval().set(num, ctx.interval_item().getText());
stackPush(new Var(interval));
return 0;
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/main/java/org/apache/hive/hplsql/Expression.java
----------------------------------------------------------------------
diff --git a/hplsql/src/main/java/org/apache/hive/hplsql/Expression.java b/hplsql/src/main/java/org/apache/hive/hplsql/Expression.java
index 7c500a8..33ef490 100644
--- a/hplsql/src/main/java/org/apache/hive/hplsql/Expression.java
+++ b/hplsql/src/main/java/org/apache/hive/hplsql/Expression.java
@@ -77,6 +77,16 @@ public class Expression {
sql.append(evalPop(ctx.expr(0)).toString());
sql.append(")");
}
+ else if (ctx.T_MUL() != null) {
+ sql.append(evalPop(ctx.expr(0)).toString());
+ sql.append(" * ");
+ sql.append(evalPop(ctx.expr(1)).toString());
+ }
+ else if (ctx.T_DIV() != null) {
+ sql.append(evalPop(ctx.expr(0)).toString());
+ sql.append(" / ");
+ sql.append(evalPop(ctx.expr(1)).toString());
+ }
else if (ctx.T_ADD() != null) {
sql.append(evalPop(ctx.expr(0)).toString());
sql.append(" + ");
@@ -325,6 +335,9 @@ public class Expression {
else if (v1.type == Type.DATE && v2.type == Type.BIGINT) {
exec.stackPush(changeDateByInt((Date)v1.value, (Long)v2.value, true /*add*/));
}
+ else if (v1.type == Type.STRING && v2.type == Type.STRING) {
+ exec.stackPush(((String)v1.value) + ((String)v2.value));
+ }
else if (v1.type == Type.DATE && v2.type == Type.INTERVAL) {
exec.stackPush(new Var(((Interval)v2.value).dateChange((Date)v1.value, true /*add*/)));
}
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/main/java/org/apache/hive/hplsql/File.java
----------------------------------------------------------------------
diff --git a/hplsql/src/main/java/org/apache/hive/hplsql/File.java b/hplsql/src/main/java/org/apache/hive/hplsql/File.java
index e748772..d18c604 100644
--- a/hplsql/src/main/java/org/apache/hive/hplsql/File.java
+++ b/hplsql/src/main/java/org/apache/hive/hplsql/File.java
@@ -83,6 +83,17 @@ public class File {
e.printStackTrace();
}
}
+
+ /**
+ * Check if the directory or file exists
+ * @throws IOException
+ */
+ boolean exists(String name) throws IOException {
+ if (fs == null) {
+ fs = createFs();
+ }
+ return fs.exists(new Path(name));
+ }
/**
* Read a character from input
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/main/java/org/apache/hive/hplsql/Ftp.java
----------------------------------------------------------------------
diff --git a/hplsql/src/main/java/org/apache/hive/hplsql/Ftp.java b/hplsql/src/main/java/org/apache/hive/hplsql/Ftp.java
new file mode 100644
index 0000000..1f2fc5c
--- /dev/null
+++ b/hplsql/src/main/java/org/apache/hive/hplsql/Ftp.java
@@ -0,0 +1,415 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.hplsql;
+
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Hashtable;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.regex.Pattern;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.commons.net.ftp.FTP;
+import org.apache.commons.net.ftp.FTPClient;
+import org.apache.commons.net.ftp.FTPFile;
+import org.antlr.v4.runtime.ParserRuleContext;
+
+public class Ftp implements Runnable {
+ String host;
+ String user;
+ String pwd;
+ String dir;
+ String targetDir;
+ String filePattern;
+ boolean subdir = false;
+ boolean local = false;
+ boolean newOnly = false;
+ int sessions = 1;
+
+ int fileCnt = 0;
+ int dirCnt = 0;
+ long ftpSizeInBytes = 0;
+
+ FTPClient ftp = null;
+ ConcurrentLinkedQueue<String> filesQueue = new ConcurrentLinkedQueue<String>();
+ Hashtable<String, FTPFile> filesMap = new Hashtable<String, FTPFile>();
+
+ AtomicInteger currentFileCnt = new AtomicInteger(1);
+ AtomicInteger currentThreadCnt = new AtomicInteger(0);
+ AtomicInteger fileCntSuccess = new AtomicInteger(0);
+ AtomicLong bytesTransferredAll = new AtomicLong(0);
+
+ Exec exec;
+ boolean trace = false;
+ boolean info = false;
+
+ Ftp(Exec e) {
+ exec = e;
+ trace = exec.getTrace();
+ info = exec.getInfo();
+ }
+
+ /**
+ * Run COPY FROM FTP command
+ */
+ Integer run(HplsqlParser.Copy_from_ftp_stmtContext ctx) {
+ trace(ctx, "COPY FROM FTP");
+ initOptions(ctx);
+ ftp = openConnection(ctx);
+ if (ftp != null) {
+ Timer timer = new Timer();
+ long start = timer.start();
+ if (info) {
+ info(ctx, "Retrieving directory listing");
+ }
+ retrieveFileList(dir);
+ long elapsed = timer.stop();
+ if (info) {
+ info(ctx, "Files to copy: " + Utils.formatSizeInBytes(ftpSizeInBytes) + ", " + Utils.formatCnt(fileCnt, "file") + ", " + Utils.formatCnt(dirCnt, "subdirectory", "subdirectories") + " scanned (" + timer.format() + ")");
+ }
+ if (fileCnt > 0) {
+ copyFiles(ctx);
+ }
+ }
+ return 0;
+ }
+
+ /**
+ * Copy the specified files from FTP
+ */
+ void copyFiles(HplsqlParser.Copy_from_ftp_stmtContext ctx) {
+ Timer timer = new Timer();
+ long start = timer.start();
+ if (fileCnt > 1 && sessions > 1) {
+ if (sessions > fileCnt) {
+ sessions = fileCnt;
+ }
+ try {
+ Thread threads[] = new Thread[sessions];
+ for (int i = 0; i < sessions; i++) {
+ threads[i] = new Thread(this);
+ threads[i].start();
+ }
+ for (int i = 0; i < sessions; i++) {
+ threads[i].join();
+ }
+ }
+ catch(Exception e) {
+ }
+ }
+ else { // Transfer files in the single session
+ run();
+ }
+ if (info) {
+ long elapsed = timer.stop();
+ long bytesAll = bytesTransferredAll.get();
+ info(ctx, "Transfer complete: " + Utils.formatSizeInBytes(bytesAll) + ", " + fileCntSuccess.get() + " files ok, " + (fileCnt - fileCntSuccess.get()) + " failed, "+ Utils.formatTime(elapsed) + ", " + Utils.formatBytesPerSec(bytesAll, elapsed));
+ }
+ }
+
+ /**
+ * Run a thread to transfer files
+ */
+ public void run() {
+ byte[] data = null;
+ Timer timer = new Timer();
+ FTPClient ftp = this.ftp;
+ if (currentThreadCnt.getAndIncrement() > 0) {
+ ftp = openConnection(null);
+ }
+ while(true) {
+ String file = filesQueue.poll();
+ if (file == null) {
+ break;
+ }
+ int num = currentFileCnt.getAndIncrement();
+ FTPFile ftpFile = filesMap.get(file);
+ long ftpSizeInBytes = ftpFile.getSize();
+ String fmtSizeInBytes = Utils.formatSizeInBytes(ftpSizeInBytes);
+ String targetFile = getTargetFileName(file);
+ if (info) {
+ info(null, " " + file + " - started (" + num + " of " + fileCnt + ", " + fmtSizeInBytes +")");
+ }
+ try {
+ InputStream in = ftp.retrieveFileStream(file);
+ OutputStream out = null;
+ java.io.File targetLocalFile = null;
+ File targetHdfsFile = null;
+ if (local) {
+ targetLocalFile = new java.io.File(targetFile);
+ if (!targetLocalFile.exists()) {
+ targetLocalFile.getParentFile().mkdirs();
+ targetLocalFile.createNewFile();
+ }
+ out = new FileOutputStream(targetLocalFile, false /*append*/);
+ }
+ else {
+ targetHdfsFile = new File();
+ out = targetHdfsFile.create(targetFile, true /*overwrite*/);
+ }
+ if (data == null) {
+ data = new byte[3*1024*1024];
+ }
+ int bytesRead = -1;
+ long bytesReadAll = 0;
+ long start = timer.start();
+ long prev = start;
+ long readTime = 0;
+ long writeTime = 0;
+ long cur, cur2, cur3;
+ while (true) {
+ cur = timer.current();
+ bytesRead = in.read(data);
+ cur2 = timer.current();
+ readTime += (cur2 - cur);
+ if (bytesRead == -1) {
+ break;
+ }
+ out.write(data, 0, bytesRead);
+ out.flush();
+ cur3 = timer.current();
+ writeTime += (cur3 - cur2);
+ bytesReadAll += bytesRead;
+ if (info) {
+ cur = timer.current();
+ if (cur - prev > 13000) {
+ long elapsed = cur - start;
+ info(null, " " + file + " - in progress (" + Utils.formatSizeInBytes(bytesReadAll) + " of " + fmtSizeInBytes + ", " + Utils.formatPercent(bytesReadAll, ftpSizeInBytes) + ", " + Utils.formatTime(elapsed) + ", " + Utils.formatBytesPerSec(bytesReadAll, elapsed) + ", " + Utils.formatBytesPerSec(bytesReadAll, readTime) + " read, " + Utils.formatBytesPerSec(bytesReadAll, writeTime) + " write)");
+ prev = cur;
+ }
+ }
+ }
+ if (ftp.completePendingCommand()) {
+ in.close();
+ cur = timer.current();
+ out.close();
+ readTime += (timer.current() - cur);
+ bytesTransferredAll.addAndGet(bytesReadAll);
+ fileCntSuccess.incrementAndGet();
+ if (info) {
+ long elapsed = timer.stop();
+ info(null, " " + file + " - complete (" + Utils.formatSizeInBytes(bytesReadAll) + ", " + Utils.formatTime(elapsed) + ", " + Utils.formatBytesPerSec(bytesReadAll, elapsed) + ", " + Utils.formatBytesPerSec(bytesReadAll, readTime) + " read, " + Utils.formatBytesPerSec(bytesReadAll, writeTime) + " write)");
+ }
+ }
+ else {
+ in.close();
+ out.close();
+ if (info) {
+ info(null, " " + file + " - failed");
+ }
+ exec.signal(Signal.Type.SQLEXCEPTION, "File transfer failed: " + file);
+ }
+ }
+ catch(IOException e) {
+ exec.signal(e);
+ }
+ }
+ try {
+ if (ftp.isConnected()) {
+ ftp.logout();
+ ftp.disconnect();
+ }
+ }
+ catch (IOException e) {
+ }
+ }
+
+ /**
+ * Get the list of files to transfer
+ */
+ void retrieveFileList(String dir) {
+ if (info) {
+ if (dir == null || dir.isEmpty()) {
+ info(null, " Listing the current working FTP directory");
+ }
+ else {
+ info(null, " Listing " + dir);
+ }
+ }
+ try {
+ FTPFile[] files = ftp.listFiles(dir);
+ ArrayList<FTPFile> dirs = new ArrayList<FTPFile>();
+ for (FTPFile file : files) {
+ String name = file.getName();
+ if (file.isFile()) {
+ if (filePattern == null || Pattern.matches(filePattern, name)) {
+ if (dir != null && !dir.isEmpty()) {
+ name = dir + "/" + name;
+ }
+ if (!newOnly || !isTargetExists(name)) {
+ fileCnt++;
+ ftpSizeInBytes += file.getSize();
+ filesQueue.add(name);
+ filesMap.put(name, file);
+ }
+ }
+ }
+ else {
+ if (subdir && !name.equals(".") && !name.equals("..")) {
+ dirCnt++;
+ dirs.add(file);
+ }
+ }
+ }
+ if (subdir) {
+ for (FTPFile d : dirs) {
+ String sd = d.getName();
+ if (dir != null && !dir.isEmpty()) {
+ sd = dir + "/" + sd;
+ }
+ retrieveFileList(sd);
+ }
+ }
+ }
+ catch (IOException e) {
+ exec.signal(e);
+ }
+ }
+
+ /**
+ * Open and initialize FTP
+ */
+ FTPClient openConnection(HplsqlParser.Copy_from_ftp_stmtContext ctx) {
+ FTPClient ftp = new FTPClient();
+ Timer timer = new Timer();
+ long start = timer.start();
+ try {
+ ftp.connect(host);
+ ftp.enterLocalPassiveMode();
+ ftp.setFileType(FTP.BINARY_FILE_TYPE);
+ if (!ftp.login(user, pwd)) {
+ if (ftp.isConnected()) {
+ ftp.disconnect();
+ }
+ exec.signal(Signal.Type.SQLEXCEPTION, "Cannot login to FTP server: " + host);
+ return null;
+ }
+ long elapsed = timer.stop();
+ if (info) {
+ info(ctx, "Connected to ftp: " + host + " (" + timer.format() + ")");
+ }
+ }
+ catch (IOException e) {
+ exec.signal(e);
+ }
+ return ftp;
+ }
+
+ /**
+ * Check if the file already exists in the target file system
+ */
+ boolean isTargetExists(String name) {
+ String target = getTargetFileName(name);
+ try {
+ if (local) {
+ if (new java.io.File(target).exists()) {
+ return true;
+ }
+ }
+ else if (new File().exists(target)) {
+ return true;
+ }
+ }
+ catch(Exception e) {
+ }
+ return false;
+ }
+
+ /**
+ * Get the target file relative path and name
+ */
+ String getTargetFileName(String file) {
+ int len = dir.length();
+ return targetDir + file.substring(len);
+ }
+
+ /**
+ * Initialize COPY FROM FTP command options
+ */
+ void initOptions(HplsqlParser.Copy_from_ftp_stmtContext ctx) {
+ host = evalPop(ctx.expr()).toString();
+ int cnt = ctx.copy_ftp_option().size();
+ for (int i = 0; i < cnt; i++) {
+ HplsqlParser.Copy_ftp_optionContext option = ctx.copy_ftp_option(i);
+ if (option.T_USER() != null) {
+ user = evalPop(option.expr()).toString();
+ }
+ else if (option.T_PWD() != null) {
+ pwd = evalPop(option.expr()).toString();
+ }
+ else if (option.T_DIR() != null) {
+ if (option.file_name() != null) {
+ dir = option.file_name().getText();
+ }
+ else {
+ dir = evalPop(option.expr()).toString();
+ }
+ }
+ else if (option.T_FILES() != null) {
+ filePattern = evalPop(option.expr()).toString();
+ }
+ else if (option.T_NEW() != null) {
+ newOnly = true;
+ }
+ else if (option.T_SUBDIR() != null) {
+ subdir = true;
+ }
+ else if (option.T_SESSIONS() != null) {
+ sessions = evalPop(option.expr()).intValue();
+ }
+ else if (option.T_TO() != null) {
+ if (option.file_name() != null) {
+ targetDir = option.file_name().getText();
+ }
+ else {
+ targetDir = evalPop(option.expr()).toString();
+ }
+ if (option.T_LOCAL() != null) {
+ local = true;
+ }
+ }
+ }
+ }
+
+ /**
+ * Evaluate the expression and pop value from the stack
+ */
+ Var evalPop(ParserRuleContext ctx) {
+ exec.visit(ctx);
+ if (!exec.stack.isEmpty()) {
+ return exec.stackPop();
+ }
+ return Var.Empty;
+ }
+
+ /**
+ * Trace and information
+ */
+ public void trace(ParserRuleContext ctx, String message) {
+ exec.trace(ctx, message);
+ }
+
+ public void info(ParserRuleContext ctx, String message) {
+ exec.info(ctx, message);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/main/java/org/apache/hive/hplsql/Meta.java
----------------------------------------------------------------------
diff --git a/hplsql/src/main/java/org/apache/hive/hplsql/Meta.java b/hplsql/src/main/java/org/apache/hive/hplsql/Meta.java
index 2e04ef9..46bd55a 100644
--- a/hplsql/src/main/java/org/apache/hive/hplsql/Meta.java
+++ b/hplsql/src/main/java/org/apache/hive/hplsql/Meta.java
@@ -139,15 +139,26 @@ public class Meta {
}
/**
- * Normalize identifier name (convert "" [] to `` i.e.)
+ * Normalize identifier for a database object (convert "" [] to `` i.e.)
*/
- public String normalizeIdentifier(String name) {
+ public String normalizeObjectIdentifier(String name) {
ArrayList<String> parts = splitIdentifier(name);
- if (parts != null) {
+ if (parts != null) { // more then one part exist
StringBuilder norm = new StringBuilder();
- for (int i = 0; i < parts.size(); i++) {
- norm.append(normalizeIdentifierPart(parts.get(i)));
- if (i + 1 < parts.size()) {
+ int size = parts.size();
+ boolean appended = false;
+ for (int i = 0; i < size; i++) {
+ if (i == size - 2) { // schema name
+ String schema = getTargetSchemaName(parts.get(i));
+ if (schema != null) {
+ norm.append(schema);
+ appended = true;
+ }
+ } else {
+ norm.append(normalizeIdentifierPart(parts.get(i)));
+ appended = true;
+ }
+ if (i + 1 < parts.size() && appended) {
norm.append(".");
}
}
@@ -157,7 +168,17 @@ public class Meta {
}
/**
- * Normalize identifier (single part)
+ * Get the schema name to be used in the final executed SQL
+ */
+ String getTargetSchemaName(String name) {
+ if (name.equalsIgnoreCase("dbo") || name.equalsIgnoreCase("[dbo]")) {
+ return null;
+ }
+ return normalizeIdentifierPart(name);
+ }
+
+ /**
+ * Normalize identifier (single part) - convert "" [] to `` i.e.
*/
public String normalizeIdentifierPart(String name) {
char start = name.charAt(0);
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/main/java/org/apache/hive/hplsql/Package.java
----------------------------------------------------------------------
diff --git a/hplsql/src/main/java/org/apache/hive/hplsql/Package.java b/hplsql/src/main/java/org/apache/hive/hplsql/Package.java
index 84fa94a..15be59c 100644
--- a/hplsql/src/main/java/org/apache/hive/hplsql/Package.java
+++ b/hplsql/src/main/java/org/apache/hive/hplsql/Package.java
@@ -150,6 +150,9 @@ public class Package {
HashMap<String, Var> out = new HashMap<String, Var>();
exec.enterScope(Scope.Type.ROUTINE, this);
exec.callStackPush(name);
+ if (p.declare_block_inplace() != null) {
+ visit(p.declare_block_inplace());
+ }
if (p.create_routine_params() != null) {
function.setCallParameters(ctx, actualParams, p.create_routine_params(), out);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/main/java/org/apache/hive/hplsql/Row.java
----------------------------------------------------------------------
diff --git a/hplsql/src/main/java/org/apache/hive/hplsql/Row.java b/hplsql/src/main/java/org/apache/hive/hplsql/Row.java
index deeacaf..91392c7 100644
--- a/hplsql/src/main/java/org/apache/hive/hplsql/Row.java
+++ b/hplsql/src/main/java/org/apache/hive/hplsql/Row.java
@@ -46,14 +46,14 @@ public class Row {
void addColumn(String name, String type) {
Column column = new Column(name, type);
columns.add(column);
- columnMap.put(name, column);
+ columnMap.put(name.toUpperCase(), column);
}
/**
* Get the data type by column name
*/
String getType(String name) {
- Column column = columnMap.get(name);
+ Column column = columnMap.get(name.toUpperCase());
if (column != null) {
return column.getType();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/main/java/org/apache/hive/hplsql/Select.java
----------------------------------------------------------------------
diff --git a/hplsql/src/main/java/org/apache/hive/hplsql/Select.java b/hplsql/src/main/java/org/apache/hive/hplsql/Select.java
index 56fbb05..4bee252 100644
--- a/hplsql/src/main/java/org/apache/hive/hplsql/Select.java
+++ b/hplsql/src/main/java/org/apache/hive/hplsql/Select.java
@@ -202,7 +202,7 @@ public class Select {
sql.append(" " + evalPop(ctx.select_list()));
if (ctx.from_clause() != null) {
sql.append(" " + evalPop(ctx.from_clause()));
- } else {
+ } else if (conf.dualTable != null) {
sql.append(" FROM " + conf.dualTable);
}
if (ctx.where_clause() != null) {
@@ -342,7 +342,9 @@ public class Select {
sql.append(", ");
}
}
- sql.append(" FROM " + conf.dualTable);
+ if (conf.dualTable != null) {
+ sql.append(" FROM " + conf.dualTable);
+ }
if (i + 1 < rows) {
sql.append("\nUNION ALL\n");
}
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/main/java/org/apache/hive/hplsql/Stmt.java
----------------------------------------------------------------------
diff --git a/hplsql/src/main/java/org/apache/hive/hplsql/Stmt.java b/hplsql/src/main/java/org/apache/hive/hplsql/Stmt.java
index e6ac196..d35f994 100644
--- a/hplsql/src/main/java/org/apache/hive/hplsql/Stmt.java
+++ b/hplsql/src/main/java/org/apache/hive/hplsql/Stmt.java
@@ -191,6 +191,17 @@ public class Stmt {
evalString(sql);
return 0;
}
+
+ /**
+ * CREATE TABLE options for MySQL
+ */
+ public Integer createTableMysqlOptions(HplsqlParser.Create_table_options_mysql_itemContext ctx) {
+ StringBuilder sql = new StringBuilder();
+ if (ctx.T_COMMENT() != null) {
+ evalString(ctx.T_COMMENT().getText() + " " + evalPop(ctx.expr()).toSqlString());
+ }
+ return 0;
+ }
/**
* DECLARE TEMPORARY TABLE statement
@@ -204,6 +215,44 @@ public class Stmt {
}
/**
+ * CREATE DATABASE | SCHEMA statement
+ */
+ public Integer createDatabase(HplsqlParser.Create_database_stmtContext ctx) {
+ trace(ctx, "CREATE DATABASE");
+ StringBuilder sql = new StringBuilder();
+ sql.append(ctx.T_CREATE().getText() + " ");
+ if (ctx.T_DATABASE() != null) {
+ sql.append(ctx.T_DATABASE().getText() + " ");
+ }
+ else {
+ sql.append(ctx.T_SCHEMA().getText() + " ");
+ }
+ if (ctx.T_IF() != null) {
+ sql.append(exec.getText(ctx, ctx.T_IF().getSymbol(), ctx.T_EXISTS().getSymbol()) + " ");
+ }
+ sql.append(evalPop(ctx.expr()).toString());
+ int cnt = ctx.create_database_option().size();
+ for (int i = 0; i < cnt; i++) {
+ HplsqlParser.Create_database_optionContext option = ctx.create_database_option(i);
+ if (option.T_COMMENT() != null) {
+ sql.append(" " + option.T_COMMENT().getText() + " " + evalPop(option.expr()).toSqlString());
+ }
+ else if (option.T_LOCATION() != null) {
+ sql.append(" " + option.T_LOCATION().getText() + " " + evalPop(option.expr()).toSqlString());
+ }
+ }
+ trace(ctx, sql.toString());
+ Query query = exec.executeSql(ctx, sql.toString(), exec.conf.defaultConnection);
+ if (query.error()) {
+ exec.signal(query);
+ return 1;
+ }
+ exec.setSqlSuccess();
+ exec.closeQuery(query, exec.conf.defaultConnection);
+ return 0;
+ }
+
+ /**
* CREATE LOCAL TEMPORARY | VOLATILE TABLE statement
*/
public Integer createLocalTemporaryTable(HplsqlParser.Create_local_temp_table_stmtContext ctx) {
@@ -252,6 +301,45 @@ public class Stmt {
}
/**
+ * DESCRIBE statement
+ */
+ public Integer describe(HplsqlParser.Describe_stmtContext ctx) {
+ trace(ctx, "DESCRIBE");
+ String sql = "DESCRIBE " + evalPop(ctx.table_name()).toString();
+ trace(ctx, sql);
+ Query query = exec.executeSql(ctx, sql, exec.conf.defaultConnection);
+ if (query.error()) {
+ exec.signal(query);
+ return 1;
+ }
+ try {
+ ResultSet rs = query.getResultSet();
+ ResultSetMetaData rm = null;
+ if (rs != null) {
+ rm = rs.getMetaData();
+ int cols = rm.getColumnCount();
+ while (rs.next()) {
+ for (int i = 1; i <= cols; i++) {
+ if (i > 1) {
+ System.out.print("\t");
+ }
+ System.out.print(rs.getString(i));
+ }
+ System.out.println("");
+ }
+ }
+ }
+ catch (SQLException e) {
+ exec.signal(query);
+ exec.closeQuery(query, exec.conf.defaultConnection);
+ return 1;
+ }
+ exec.setSqlSuccess();
+ exec.closeQuery(query, exec.conf.defaultConnection);
+ return 0;
+ }
+
+ /**
* DROP statement
*/
public Integer drop(HplsqlParser.Drop_stmtContext ctx) {
@@ -260,10 +348,17 @@ public class Stmt {
if (ctx.T_TABLE() != null) {
sql = "DROP TABLE ";
if (ctx.T_EXISTS() != null) {
- sql += "IF NOT EXISTS ";
+ sql += "IF EXISTS ";
}
sql += evalPop(ctx.table_name()).toString();
}
+ else if (ctx.T_DATABASE() != null || ctx.T_SCHEMA() != null) {
+ sql = "DROP DATABASE ";
+ if (ctx.T_EXISTS() != null) {
+ sql += "IF EXISTS ";
+ }
+ sql += evalPop(ctx.expr()).toString();
+ }
if (sql != null) {
trace(ctx, sql);
Query query = exec.executeSql(ctx, sql, exec.conf.defaultConnection);
@@ -278,6 +373,23 @@ public class Stmt {
}
/**
+ * TRUNCATE statement
+ */
+ public Integer truncate(HplsqlParser.Truncate_stmtContext ctx) {
+ trace(ctx, "TRUNCATE");
+ String sql = "TRUNCATE TABLE " + evalPop(ctx.table_name()).toString();
+ trace(ctx, sql);
+ Query query = exec.executeSql(ctx, sql, exec.conf.defaultConnection);
+ if (query.error()) {
+ exec.signal(query);
+ return 1;
+ }
+ exec.setSqlSuccess();
+ exec.closeQuery(query, exec.conf.defaultConnection);
+ return 0;
+ }
+
+ /**
* OPEN cursor statement
*/
public Integer open(HplsqlParser.Open_stmtContext ctx) {
@@ -563,11 +675,21 @@ public class Stmt {
*/
public Integer insertSelect(HplsqlParser.Insert_stmtContext ctx) {
trace(ctx, "INSERT SELECT");
- String table = evalPop(ctx.table_name()).toString();
- String select = evalPop(ctx.select_stmt()).toString();
- String sql = "INSERT INTO TABLE " + table + " " + select;
- trace(ctx, sql);
- Query query = exec.executeSql(ctx, sql, exec.conf.defaultConnection);
+ StringBuilder sql = new StringBuilder();
+ sql.append(ctx.T_INSERT().getText() + " ");
+ if (ctx.T_OVERWRITE() != null) {
+ sql.append(ctx.T_OVERWRITE().getText() + " " + ctx.T_TABLE().getText() + " ");
+ }
+ else {
+ sql.append(ctx.T_INTO().getText() + " ");
+ if (ctx.T_TABLE() != null) {
+ sql.append(ctx.T_TABLE().getText() + " ");
+ }
+ }
+ sql.append(evalPop(ctx.table_name()).toString() + " ");
+ sql.append(evalPop(ctx.select_stmt()).toString());
+ trace(ctx, sql.toString());
+ Query query = exec.executeSql(ctx, sql.toString(), exec.conf.defaultConnection);
if (query.error()) {
exec.signal(query);
return 1;
@@ -621,7 +743,9 @@ public class Stmt {
}
}
else if (type == Conn.Type.HIVE && conf.insertValues == Conf.InsertValues.SELECT) {
- sql.append(" FROM " + conf.dualTable);
+ if (conf.dualTable != null) {
+ sql.append(" FROM " + conf.dualTable);
+ }
if (i + 1 < rows) {
sql.append("\nUNION ALL\n");
}
@@ -641,6 +765,30 @@ public class Stmt {
}
/**
+ * INSERT DIRECTORY statement
+ */
+ public Integer insertDirectory(HplsqlParser.Insert_directory_stmtContext ctx) {
+ trace(ctx, "INSERT DIRECTORY");
+ StringBuilder sql = new StringBuilder();
+ sql.append(ctx.T_INSERT().getText() + " " + ctx.T_OVERWRITE().getText() + " ");
+ if (ctx.T_LOCAL() != null) {
+ sql.append(ctx.T_LOCAL().getText() + " ");
+ }
+ sql.append(ctx.T_DIRECTORY().getText() + " " + evalPop(ctx.expr_file()).toSqlString() + " ");
+ sql.append(evalPop(ctx.expr_select()).toString());
+ String conn = exec.getStatementConnection();
+ trace(ctx, sql.toString());
+ Query query = exec.executeSql(ctx, sql.toString(), conn);
+ if (query.error()) {
+ exec.signal(query);
+ return 1;
+ }
+ exec.setSqlSuccess();
+ exec.closeQuery(query, conn);
+ return 0;
+ }
+
+ /**
* GET DIAGNOSTICS EXCEPTION statement
*/
public Integer getDiagnosticsException(HplsqlParser.Get_diag_stmt_exception_itemContext ctx) {
@@ -876,7 +1024,7 @@ public class Stmt {
public Boolean execProc(HplsqlParser.Exec_stmtContext ctx) {
String name = evalPop(ctx.expr()).toString();
if (exec.function.isProc(name)) {
- if (exec.function.execProc(name, ctx.expr_func_params())) {
+ if (exec.function.execProc(name, ctx.expr_func_params(), ctx)) {
return true;
}
}
@@ -997,8 +1145,7 @@ public class Stmt {
public Integer print(HplsqlParser.Print_stmtContext ctx) {
trace(ctx, "PRINT");
if (ctx.expr() != null) {
- visit(ctx.expr());
- System.out.println(stack.pop().toString());
+ System.out.println(evalPop(ctx.expr()).toString());
}
return 0;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/main/java/org/apache/hive/hplsql/Utils.java
----------------------------------------------------------------------
diff --git a/hplsql/src/main/java/org/apache/hive/hplsql/Utils.java b/hplsql/src/main/java/org/apache/hive/hplsql/Utils.java
index 1815deb..fb60b22 100644
--- a/hplsql/src/main/java/org/apache/hive/hplsql/Utils.java
+++ b/hplsql/src/main/java/org/apache/hive/hplsql/Utils.java
@@ -280,14 +280,54 @@ public class Utils {
}
/**
+ * Format elasped time
+ */
+ public static String formatTime(long msElapsed) {
+ if (msElapsed < 60000) {
+ return msElapsed/1000 + " sec";
+ }
+ else if (msElapsed < 60000 * 60) {
+ return msElapsed/60000 + " min " + (msElapsed%60000)/1000 + " sec";
+ }
+ return "";
+ }
+
+ /**
* Format bytes per second rate
*/
public static String formatBytesPerSec(long bytes, long msElapsed) {
+ if (msElapsed < 30) {
+ return "n/a";
+ }
float bytesPerSec = ((float)bytes)/msElapsed*1000;
return Utils.formatSizeInBytes((long)bytesPerSec, "/sec");
}
/**
+ * Format percentage
+ */
+ public static String formatPercent(long current, long all) {
+ return String.format("%.1f", ((float)current)/all*100) + "%";
+ }
+
+ /**
+ * Format count
+ */
+ public static String formatCnt(long value, String suffix) {
+ if (value == 1) {
+ return value + " " + suffix;
+ }
+ return value + " " + suffix + "s";
+ }
+
+ public static String formatCnt(long value, String suffix, String suffix2) {
+ if (value == 1) {
+ return value + " " + suffix;
+ }
+ return value + " " + suffix2;
+ }
+
+ /**
* Note. This stub is to resolve name conflict with ANTLR generated source using org.antlr.v4.runtime.misc.Utils.join
*/
static <T> String join(T[] array, String separator) {
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/main/java/org/apache/hive/hplsql/Var.java
----------------------------------------------------------------------
diff --git a/hplsql/src/main/java/org/apache/hive/hplsql/Var.java b/hplsql/src/main/java/org/apache/hive/hplsql/Var.java
index 5f7b355..63a1f43 100644
--- a/hplsql/src/main/java/org/apache/hive/hplsql/Var.java
+++ b/hplsql/src/main/java/org/apache/hive/hplsql/Var.java
@@ -311,11 +311,12 @@ public class Var {
else if (type.equalsIgnoreCase("INT") || type.equalsIgnoreCase("INTEGER") || type.equalsIgnoreCase("BIGINT") ||
type.equalsIgnoreCase("SMALLINT") || type.equalsIgnoreCase("TINYINT") ||
type.equalsIgnoreCase("BINARY_INTEGER") || type.equalsIgnoreCase("PLS_INTEGER") ||
- type.equalsIgnoreCase("SIMPLE_INTEGER")) {
+ type.equalsIgnoreCase("SIMPLE_INTEGER") || type.equalsIgnoreCase("INT2") ||
+ type.equalsIgnoreCase("INT4") || type.equalsIgnoreCase("INT8")) {
return Type.BIGINT;
}
- else if (type.equalsIgnoreCase("CHAR") || type.equalsIgnoreCase("VARCHAR") || type.equalsIgnoreCase("STRING") ||
- type.equalsIgnoreCase("XML")) {
+ else if (type.equalsIgnoreCase("CHAR") || type.equalsIgnoreCase("VARCHAR") || type.equalsIgnoreCase("VARCHAR2") ||
+ type.equalsIgnoreCase("STRING") || type.equalsIgnoreCase("XML")) {
return Type.STRING;
}
else if (type.equalsIgnoreCase("DEC") || type.equalsIgnoreCase("DECIMAL") || type.equalsIgnoreCase("NUMERIC") ||
@@ -487,7 +488,10 @@ public class Var {
if (type == Type.BIGINT) {
return ((Long)value).intValue();
}
- return -1;
+ else if (type == Type.STRING) {
+ return Integer.parseInt((String)value);
+ }
+ throw new NumberFormatException();
}
/**
@@ -497,7 +501,7 @@ public class Var {
if (type == Type.BIGINT) {
return ((Long)value).longValue();
}
- return -1;
+ throw new NumberFormatException();
}
/**
@@ -507,7 +511,7 @@ public class Var {
if (type == Type.DECIMAL) {
return (BigDecimal)value;
}
- return null;
+ throw new NumberFormatException();
}
/**
@@ -523,7 +527,7 @@ public class Var {
else if (type == Type.DECIMAL) {
return ((BigDecimal)value).doubleValue();
}
- return -1;
+ throw new NumberFormatException();
}
/**
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/main/java/org/apache/hive/hplsql/functions/Function.java
----------------------------------------------------------------------
diff --git a/hplsql/src/main/java/org/apache/hive/hplsql/functions/Function.java b/hplsql/src/main/java/org/apache/hive/hplsql/functions/Function.java
index 70ef995..ed60d23 100644
--- a/hplsql/src/main/java/org/apache/hive/hplsql/functions/Function.java
+++ b/hplsql/src/main/java/org/apache/hive/hplsql/functions/Function.java
@@ -72,7 +72,7 @@ public class Function {
if (execUser(name, ctx)) {
return;
}
- else if (isProc(name) && execProc(name, ctx)) {
+ else if (isProc(name) && execProc(name, ctx, null)) {
return;
}
if (name.indexOf(".") != -1) { // Name can be qualified and spaces are allowed between parts
@@ -86,7 +86,7 @@ public class Function {
}
name = str.toString();
}
- if (trace && ctx.parent.parent instanceof HplsqlParser.Expr_stmtContext) {
+ if (trace && ctx != null && ctx.parent != null && ctx.parent.parent instanceof HplsqlParser.Expr_stmtContext) {
trace(ctx, "FUNC " + name);
}
FuncCommand func = map.get(name.toUpperCase());
@@ -219,19 +219,22 @@ public class Function {
/**
* Execute a stored procedure using CALL or EXEC statement passing parameters
*/
- public boolean execProc(String name, HplsqlParser.Expr_func_paramsContext ctx) {
+ public boolean execProc(String name, HplsqlParser.Expr_func_paramsContext ctx, ParserRuleContext callCtx) {
if (trace) {
- trace(ctx, "EXEC PROCEDURE " + name);
+ trace(callCtx, "EXEC PROCEDURE " + name);
}
HplsqlParser.Create_procedure_stmtContext procCtx = procMap.get(name.toUpperCase());
if (procCtx == null) {
- trace(ctx, "Procedure not found");
+ trace(callCtx, "Procedure not found");
return false;
}
ArrayList<Var> actualParams = getActualCallParameters(ctx);
HashMap<String, Var> out = new HashMap<String, Var>();
exec.enterScope(Scope.Type.ROUTINE);
exec.callStackPush(name);
+ if (procCtx.declare_block_inplace() != null) {
+ visit(procCtx.declare_block_inplace());
+ }
if (procCtx.create_routine_params() != null) {
setCallParameters(ctx, actualParams, procCtx.create_routine_params(), out);
}
@@ -376,6 +379,16 @@ public class Function {
}
procMap.put(name.toUpperCase(), ctx);
}
+
+ /**
+ * Get the number of parameters in function call
+ */
+ public int getParamCount(HplsqlParser.Expr_func_paramsContext ctx) {
+ if (ctx == null) {
+ return 0;
+ }
+ return ctx.func_param().size();
+ }
/**
* Execute a special function
@@ -403,8 +416,6 @@ public class Function {
execMinPartDate(ctx);
} else if(ctx.T_PART_LOC() != null) {
execPartLoc(ctx);
- } else if(ctx.T_SYSDATE() != null) {
- execCurrentTimestamp(ctx, 0);
} else {
evalNull();
}
@@ -440,21 +451,6 @@ public class Function {
}
/**
- * Get the current date and time
- */
- public void execCurrentTimestamp(HplsqlParser.Expr_spec_funcContext ctx, int defPrecision) {
- trace(ctx, "CURRENT_TIMESTAMP");
- int precision = evalPop(ctx.expr(0), defPrecision).intValue();
- String format = "yyyy-MM-dd HH:mm:ss";
- if(precision > 0 && precision <= 3) {
- format += "." + StringUtils.repeat("S", precision);
- }
- SimpleDateFormat f = new SimpleDateFormat(format);
- String s = f.format(Calendar.getInstance(TimeZone.getDefault()).getTime());
- exec.stackPush(new Var(Utils.toTimestamp(s), precision));
- }
-
- /**
* Execute MAX_PART_STRING function
*/
public void execMaxPartString(HplsqlParser.Expr_spec_funcContext ctx) {
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/main/java/org/apache/hive/hplsql/functions/FunctionDatetime.java
----------------------------------------------------------------------
diff --git a/hplsql/src/main/java/org/apache/hive/hplsql/functions/FunctionDatetime.java b/hplsql/src/main/java/org/apache/hive/hplsql/functions/FunctionDatetime.java
index 4b29c59..4d48a55 100644
--- a/hplsql/src/main/java/org/apache/hive/hplsql/functions/FunctionDatetime.java
+++ b/hplsql/src/main/java/org/apache/hive/hplsql/functions/FunctionDatetime.java
@@ -21,6 +21,7 @@ package org.apache.hive.hplsql.functions;
import java.sql.Timestamp;
import java.text.SimpleDateFormat;
import java.util.Calendar;
+import java.util.Date;
import java.util.TimeZone;
import org.apache.commons.lang.StringUtils;
@@ -37,11 +38,15 @@ public class FunctionDatetime extends Function {
@Override
public void register(Function f) {
f.map.put("DATE", new FuncCommand() { public void run(HplsqlParser.Expr_func_paramsContext ctx) { date(ctx); }});
+ f.map.put("FROM_UNIXTIME", new FuncCommand() { public void run(HplsqlParser.Expr_func_paramsContext ctx) { fromUnixtime(ctx); }});
+ f.map.put("NOW", new FuncCommand() { public void run(HplsqlParser.Expr_func_paramsContext ctx) { now(ctx); }});
f.map.put("TIMESTAMP_ISO", new FuncCommand() { public void run(HplsqlParser.Expr_func_paramsContext ctx) { timestampIso(ctx); }});
f.map.put("TO_TIMESTAMP", new FuncCommand() { public void run(HplsqlParser.Expr_func_paramsContext ctx) { toTimestamp(ctx); }});
+ f.map.put("UNIX_TIMESTAMP", new FuncCommand() { public void run(HplsqlParser.Expr_func_paramsContext ctx) { unixTimestamp(ctx); }});
f.specMap.put("CURRENT_DATE", new FuncSpecCommand() { public void run(HplsqlParser.Expr_spec_funcContext ctx) { currentDate(ctx); }});
f.specMap.put("CURRENT_TIMESTAMP", new FuncSpecCommand() { public void run(HplsqlParser.Expr_spec_funcContext ctx) { currentTimestamp(ctx); }});
+ f.specMap.put("SYSDATE", new FuncSpecCommand() { public void run(HplsqlParser.Expr_spec_funcContext ctx) { currentTimestamp(ctx); }});
f.specSqlMap.put("CURRENT_DATE", new FuncSpecCommand() { public void run(HplsqlParser.Expr_spec_funcContext ctx) { currentDateSql(ctx); }});
f.specSqlMap.put("CURRENT_TIMESTAMP", new FuncSpecCommand() { public void run(HplsqlParser.Expr_spec_funcContext ctx) { currentTimestampSql(ctx); }});
@@ -114,6 +119,17 @@ public class FunctionDatetime extends Function {
var.cast(evalPop(ctx.func_param(0).expr()));
evalVar(var);
}
+
+ /**
+ * NOW() function (current date and time)
+ */
+ void now(HplsqlParser.Expr_func_paramsContext ctx) {
+ if (ctx != null) {
+ evalNull();
+ return;
+ }
+ evalVar(currentTimestamp(3));
+ }
/**
* TIMESTAMP_ISO function
@@ -148,4 +164,28 @@ public class FunctionDatetime extends Function {
evalNull();
}
}
+
+ /**
+ * FROM_UNIXTIME() function (convert seconds since 1970-01-01 00:00:00 to timestamp)
+ */
+ void fromUnixtime(HplsqlParser.Expr_func_paramsContext ctx) {
+ int cnt = getParamCount(ctx);
+ if (cnt == 0) {
+ evalNull();
+ return;
+ }
+ long epoch = evalPop(ctx.func_param(0).expr()).longValue();
+ String format = "yyyy-MM-dd HH:mm:ss";
+ if (cnt > 1) {
+ format = evalPop(ctx.func_param(1).expr()).toString();
+ }
+ evalString(new SimpleDateFormat(format).format(new Date(epoch * 1000)));
+ }
+
+ /**
+ * UNIX_TIMESTAMP() function (current date and time in seconds since 1970-01-01 00:00:00)
+ */
+ void unixTimestamp(HplsqlParser.Expr_func_paramsContext ctx) {
+ evalVar(new Var(System.currentTimeMillis()/1000));
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/main/java/org/apache/hive/hplsql/functions/FunctionString.java
----------------------------------------------------------------------
diff --git a/hplsql/src/main/java/org/apache/hive/hplsql/functions/FunctionString.java b/hplsql/src/main/java/org/apache/hive/hplsql/functions/FunctionString.java
index 731443f..9d3fd19 100644
--- a/hplsql/src/main/java/org/apache/hive/hplsql/functions/FunctionString.java
+++ b/hplsql/src/main/java/org/apache/hive/hplsql/functions/FunctionString.java
@@ -36,6 +36,7 @@ public class FunctionString extends Function {
f.map.put("LEN", new FuncCommand() { public void run(HplsqlParser.Expr_func_paramsContext ctx) { len(ctx); }});
f.map.put("LENGTH", new FuncCommand() { public void run(HplsqlParser.Expr_func_paramsContext ctx) { length(ctx); }});
f.map.put("LOWER", new FuncCommand() { public void run(HplsqlParser.Expr_func_paramsContext ctx) { lower(ctx); }});
+ f.map.put("REPLACE", new FuncCommand() { public void run(HplsqlParser.Expr_func_paramsContext ctx) { replace(ctx); }});
f.map.put("SUBSTR", new FuncCommand() { public void run(HplsqlParser.Expr_func_paramsContext ctx) { substr(ctx); }});
f.map.put("SUBSTRING", new FuncCommand() { public void run(HplsqlParser.Expr_func_paramsContext ctx) { substr(ctx); }});
f.map.put("TO_CHAR", new FuncCommand() { public void run(HplsqlParser.Expr_func_paramsContext ctx) { toChar(ctx); }});
@@ -50,7 +51,7 @@ public class FunctionString extends Function {
*/
void concat(HplsqlParser.Expr_func_paramsContext ctx) {
StringBuilder val = new StringBuilder();
- int cnt = ctx.func_param().size();
+ int cnt = getParamCount(ctx);
boolean nulls = true;
for (int i = 0; i < cnt; i++) {
Var c = evalPop(ctx.func_param(i).expr());
@@ -71,7 +72,7 @@ public class FunctionString extends Function {
* CHAR function
*/
void char_(HplsqlParser.Expr_func_paramsContext ctx) {
- int cnt = ctx.func_param().size();
+ int cnt = getParamCount(ctx);
if (cnt != 1) {
evalNull();
return;
@@ -84,7 +85,7 @@ public class FunctionString extends Function {
* INSTR function
*/
void instr(HplsqlParser.Expr_func_paramsContext ctx) {
- int cnt = ctx.func_param().size();
+ int cnt = getParamCount(ctx);
if (cnt < 2) {
evalNull();
return;
@@ -178,10 +179,25 @@ public class FunctionString extends Function {
}
/**
+ * REPLACE function
+ */
+ void replace(HplsqlParser.Expr_func_paramsContext ctx) {
+ int cnt = getParamCount(ctx);
+ if (cnt < 3) {
+ evalNull();
+ return;
+ }
+ String str = evalPop(ctx.func_param(0).expr()).toString();
+ String what = evalPop(ctx.func_param(1).expr()).toString();
+ String with = evalPop(ctx.func_param(2).expr()).toString();
+ evalString(str.replaceAll(what, with));
+ }
+
+ /**
* SUBSTR and SUBSTRING function
*/
void substr(HplsqlParser.Expr_func_paramsContext ctx) {
- int cnt = ctx.func_param().size();
+ int cnt = getParamCount(ctx);
if (cnt < 2) {
evalNull();
return;
@@ -253,7 +269,7 @@ public class FunctionString extends Function {
* TO_CHAR function
*/
void toChar(HplsqlParser.Expr_func_paramsContext ctx) {
- int cnt = ctx.func_param().size();
+ int cnt = getParamCount(ctx);
if (cnt != 1) {
evalNull();
return;
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/java/org/apache/hive/hplsql/TestHplsqlLocal.java
----------------------------------------------------------------------
diff --git a/hplsql/src/test/java/org/apache/hive/hplsql/TestHplsqlLocal.java b/hplsql/src/test/java/org/apache/hive/hplsql/TestHplsqlLocal.java
index 042bacf..80915ea 100644
--- a/hplsql/src/test/java/org/apache/hive/hplsql/TestHplsqlLocal.java
+++ b/hplsql/src/test/java/org/apache/hive/hplsql/TestHplsqlLocal.java
@@ -92,6 +92,16 @@ public class TestHplsqlLocal {
public void testCreateFunction2() throws Exception {
run("create_function2");
}
+
+ @Test
+ public void testCreateFunction3() throws Exception {
+ run("create_function3");
+ }
+
+ @Test
+ public void testCreateFunction4() throws Exception {
+ run("create_function4");
+ }
@Test
public void testCreatePackage() throws Exception {
@@ -112,6 +122,12 @@ public class TestHplsqlLocal {
public void testCreateProcedure2() throws Exception {
run("create_procedure2");
}
+
+ @Test
+ public void testCreateProcedure3() throws Exception {
+ run("create_procedure3");
+ }
+
@Test
public void testCreateProcedureNoParams() throws Exception {
run("create_procedure_no_params");
@@ -143,6 +159,11 @@ public class TestHplsqlLocal {
}
@Test
+ public void testDeclare3() throws Exception {
+ run("declare3");
+ }
+
+ @Test
public void testDeclareCondition() throws Exception {
run("declare_condition");
}
@@ -186,7 +207,7 @@ public class TestHplsqlLocal {
public void testFloat() throws Exception {
run("float");
}
-
+
@Test
public void testForRange() throws Exception {
run("for_range");
@@ -262,6 +283,11 @@ public class TestHplsqlLocal {
}
@Test
+ public void testReplace() throws Exception {
+ run("replace");
+ }
+
+ @Test
public void testReturn() throws Exception {
run("return");
}
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/java/org/apache/hive/hplsql/TestHplsqlOffline.java
----------------------------------------------------------------------
diff --git a/hplsql/src/test/java/org/apache/hive/hplsql/TestHplsqlOffline.java b/hplsql/src/test/java/org/apache/hive/hplsql/TestHplsqlOffline.java
index b9d80c7..59b7bff 100644
--- a/hplsql/src/test/java/org/apache/hive/hplsql/TestHplsqlOffline.java
+++ b/hplsql/src/test/java/org/apache/hive/hplsql/TestHplsqlOffline.java
@@ -37,6 +37,16 @@ public class TestHplsqlOffline {
public void testCreateTableMssql() throws Exception {
run("create_table_mssql");
}
+
+ @Test
+ public void testCreateTableMssql2() throws Exception {
+ run("create_table_mssql2");
+ }
+
+ @Test
+ public void testCreateTableMysql() throws Exception {
+ run("create_table_mysql");
+ }
@Test
public void testCreateTableOra() throws Exception {
@@ -44,6 +54,16 @@ public class TestHplsqlOffline {
}
@Test
+ public void testCreateTableOra2() throws Exception {
+ run("create_table_ora2");
+ }
+
+ @Test
+ public void testCreateTablePg() throws Exception {
+ run("create_table_pg");
+ }
+
+ @Test
public void testInsertMysql() throws Exception {
run("insert_mysql");
}
@@ -52,6 +72,11 @@ public class TestHplsqlOffline {
public void testSelectDb2() throws Exception {
run("select_db2");
}
+
+ @Test
+ public void testUpdate() throws Exception {
+ run("update");
+ }
/**
* Run a test file
http://git-wip-us.apache.org/repos/asf/hive/blob/39d66a43/hplsql/src/test/queries/db/create_drop_database.sql
----------------------------------------------------------------------
diff --git a/hplsql/src/test/queries/db/create_drop_database.sql b/hplsql/src/test/queries/db/create_drop_database.sql
new file mode 100644
index 0000000..d71bbbf
--- /dev/null
+++ b/hplsql/src/test/queries/db/create_drop_database.sql
@@ -0,0 +1,5 @@
+create database 'test' || replace('2016-03-03', '-', '');
+create database if not exists test1 comment 'abc' location '/users';
+
+drop database if exists 'test' || replace('2016-03-03', '-', '');
+drop database test1;
[27/50] [abbrv] hive git commit: HIVE-13111: Fix timestamp /
interval_day_time wrong results with HIVE-9862 (Matt McCline,
reviewed by Jason Dere)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/test/results/clientpositive/tez/vector_interval_arithmetic.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_interval_arithmetic.q.out b/ql/src/test/results/clientpositive/tez/vector_interval_arithmetic.q.out
new file mode 100644
index 0000000..8409a01
--- /dev/null
+++ b/ql/src/test/results/clientpositive/tez/vector_interval_arithmetic.q.out
@@ -0,0 +1,1086 @@
+PREHOOK: query: create table unique_timestamps (tsval timestamp) STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@unique_timestamps
+POSTHOOK: query: create table unique_timestamps (tsval timestamp) STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@unique_timestamps
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/timestamps.txt' OVERWRITE INTO TABLE unique_timestamps
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@unique_timestamps
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/timestamps.txt' OVERWRITE INTO TABLE unique_timestamps
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@unique_timestamps
+PREHOOK: query: create table interval_arithmetic_1 (dateval date, tsval timestamp) stored as orc
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@interval_arithmetic_1
+POSTHOOK: query: create table interval_arithmetic_1 (dateval date, tsval timestamp) stored as orc
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@interval_arithmetic_1
+PREHOOK: query: insert overwrite table interval_arithmetic_1
+ select cast(tsval as date), tsval from unique_timestamps
+PREHOOK: type: QUERY
+PREHOOK: Input: default@unique_timestamps
+PREHOOK: Output: default@interval_arithmetic_1
+POSTHOOK: query: insert overwrite table interval_arithmetic_1
+ select cast(tsval as date), tsval from unique_timestamps
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@unique_timestamps
+POSTHOOK: Output: default@interval_arithmetic_1
+POSTHOOK: Lineage: interval_arithmetic_1.dateval EXPRESSION [(unique_timestamps)unique_timestamps.FieldSchema(name:tsval, type:timestamp, comment:null), ]
+POSTHOOK: Lineage: interval_arithmetic_1.tsval SIMPLE [(unique_timestamps)unique_timestamps.FieldSchema(name:tsval, type:timestamp, comment:null), ]
+_c0 tsval
+PREHOOK: query: -- interval year-month arithmetic
+explain
+select
+ dateval,
+ dateval - interval '2-2' year to month,
+ dateval - interval '-2-2' year to month,
+ dateval + interval '2-2' year to month,
+ dateval + interval '-2-2' year to month,
+ - interval '2-2' year to month + dateval,
+ interval '2-2' year to month + dateval
+from interval_arithmetic_1
+order by dateval
+PREHOOK: type: QUERY
+POSTHOOK: query: -- interval year-month arithmetic
+explain
+select
+ dateval,
+ dateval - interval '2-2' year to month,
+ dateval - interval '-2-2' year to month,
+ dateval + interval '2-2' year to month,
+ dateval + interval '-2-2' year to month,
+ - interval '2-2' year to month + dateval,
+ interval '2-2' year to month + dateval
+from interval_arithmetic_1
+order by dateval
+POSTHOOK: type: QUERY
+Explain
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: interval_arithmetic_1
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: dateval (type: date), (dateval - 2-2) (type: date), (dateval - -2-2) (type: date), (dateval + 2-2) (type: date), (dateval + -2-2) (type: date), (-2-2 + dateval) (type: date), (2-2 + dateval) (type: date)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: date)
+ sort order: +
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: date), _col4 (type: date), _col5 (type: date), _col6 (type: date)
+ Execution mode: vectorized
+ Reducer 2
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: date), VALUE._col0 (type: date), VALUE._col1 (type: date), VALUE._col2 (type: date), VALUE._col3 (type: date), VALUE._col4 (type: date), VALUE._col5 (type: date)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select
+ dateval,
+ dateval - interval '2-2' year to month,
+ dateval - interval '-2-2' year to month,
+ dateval + interval '2-2' year to month,
+ dateval + interval '-2-2' year to month,
+ - interval '2-2' year to month + dateval,
+ interval '2-2' year to month + dateval
+from interval_arithmetic_1
+order by dateval
+PREHOOK: type: QUERY
+PREHOOK: Input: default@interval_arithmetic_1
+#### A masked pattern was here ####
+POSTHOOK: query: select
+ dateval,
+ dateval - interval '2-2' year to month,
+ dateval - interval '-2-2' year to month,
+ dateval + interval '2-2' year to month,
+ dateval + interval '-2-2' year to month,
+ - interval '2-2' year to month + dateval,
+ interval '2-2' year to month + dateval
+from interval_arithmetic_1
+order by dateval
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@interval_arithmetic_1
+#### A masked pattern was here ####
+dateval c1 c2 c3 c4 c5 c6
+0004-09-22 0002-07-22 0006-11-22 0006-11-22 0002-07-22 0002-07-22 0006-11-22
+0528-10-27 0526-08-27 0530-12-27 0530-12-27 0526-08-27 0526-08-27 0530-12-27
+1319-02-02 1316-12-02 1321-04-02 1321-04-02 1316-12-02 1316-12-02 1321-04-02
+1404-07-23 1402-05-23 1406-09-23 1406-09-23 1402-05-23 1402-05-23 1406-09-23
+1815-05-06 1813-03-06 1817-07-06 1817-07-06 1813-03-06 1813-03-06 1817-07-06
+1883-04-17 1881-02-17 1885-06-17 1885-06-17 1881-02-17 1881-02-17 1885-06-17
+1966-08-16 1964-06-16 1968-10-16 1968-10-16 1964-06-16 1964-06-16 1968-10-16
+1973-04-17 1971-02-17 1975-06-17 1975-06-17 1971-02-17 1971-02-17 1975-06-17
+1974-10-04 1972-08-04 1976-12-04 1976-12-04 1972-08-04 1972-08-04 1976-12-04
+1976-03-03 1974-01-03 1978-05-03 1978-05-03 1974-01-03 1974-01-03 1978-05-03
+1976-05-06 1974-03-06 1978-07-06 1978-07-06 1974-03-06 1974-03-06 1978-07-06
+1978-08-05 1976-06-05 1980-10-05 1980-10-05 1976-06-05 1976-06-05 1980-10-05
+1981-04-25 1979-02-25 1983-06-25 1983-06-25 1979-02-25 1979-02-25 1983-06-25
+1981-11-15 1979-09-15 1984-01-15 1984-01-15 1979-09-15 1979-09-15 1984-01-15
+1985-07-20 1983-05-20 1987-09-20 1987-09-20 1983-05-20 1983-05-20 1987-09-20
+1985-11-18 1983-09-18 1988-01-18 1988-01-18 1983-09-18 1983-09-18 1988-01-18
+1987-02-21 1984-12-21 1989-04-21 1989-04-21 1984-12-21 1984-12-21 1989-04-21
+1987-05-28 1985-03-28 1989-07-28 1989-07-28 1985-03-28 1985-03-28 1989-07-28
+1998-10-16 1996-08-16 2000-12-16 2000-12-16 1996-08-16 1996-08-16 2000-12-16
+1999-10-03 1997-08-03 2001-12-03 2001-12-03 1997-08-03 1997-08-03 2001-12-03
+2000-12-18 1998-10-18 2003-02-18 2003-02-18 1998-10-18 1998-10-18 2003-02-18
+2002-05-10 2000-03-10 2004-07-10 2004-07-10 2000-03-10 2000-03-10 2004-07-10
+2003-09-23 2001-07-23 2005-11-23 2005-11-23 2001-07-23 2001-07-23 2005-11-23
+2004-03-07 2002-01-07 2006-05-07 2006-05-07 2002-01-07 2002-01-07 2006-05-07
+2007-02-09 2004-12-09 2009-04-09 2009-04-09 2004-12-09 2004-12-09 2009-04-09
+2009-01-21 2006-11-21 2011-03-21 2011-03-21 2006-11-21 2006-11-21 2011-03-21
+2010-04-08 2008-02-08 2012-06-08 2012-06-08 2008-02-08 2008-02-08 2012-06-08
+2013-04-07 2011-02-07 2015-06-07 2015-06-07 2011-02-07 2011-02-07 2015-06-07
+2013-04-10 2011-02-10 2015-06-10 2015-06-10 2011-02-10 2011-02-10 2015-06-10
+2021-09-24 2019-07-24 2023-11-24 2023-11-24 2019-07-24 2019-07-24 2023-11-24
+2024-11-11 2022-09-11 2027-01-11 2027-01-11 2022-09-11 2022-09-11 2027-01-11
+4143-07-08 4141-05-08 4145-09-08 4145-09-08 4141-05-08 4141-05-08 4145-09-08
+4966-12-04 4964-10-04 4969-02-04 4969-02-04 4964-10-04 4964-10-04 4969-02-04
+5339-02-01 5336-12-01 5341-04-01 5341-04-01 5336-12-01 5336-12-01 5341-04-01
+5344-10-04 5342-08-04 5346-12-04 5346-12-04 5342-08-04 5342-08-04 5346-12-04
+5397-07-13 5395-05-13 5399-09-13 5399-09-13 5395-05-13 5395-05-13 5399-09-13
+5966-07-09 5964-05-09 5968-09-09 5968-09-09 5964-05-09 5964-05-09 5968-09-09
+6229-06-28 6227-04-28 6231-08-28 6231-08-28 6227-04-28 6227-04-28 6231-08-28
+6482-04-27 6480-02-27 6484-06-27 6484-06-27 6480-02-27 6480-02-27 6484-06-27
+6631-11-13 6629-09-13 6634-01-13 6634-01-13 6629-09-13 6629-09-13 6634-01-13
+6705-09-28 6703-07-28 6707-11-28 6707-11-28 6703-07-28 6703-07-28 6707-11-28
+6731-02-12 6728-12-12 6733-04-12 6733-04-12 6728-12-12 6728-12-12 6733-04-12
+7160-12-02 7158-10-02 7163-02-02 7163-02-02 7158-10-02 7158-10-02 7163-02-02
+7409-09-07 7407-07-07 7411-11-07 7411-11-07 7407-07-07 7407-07-07 7411-11-07
+7503-06-23 7501-04-23 7505-08-23 7505-08-23 7501-04-23 7501-04-23 7505-08-23
+8422-07-22 8420-05-22 8424-09-22 8424-09-22 8420-05-22 8420-05-22 8424-09-22
+8521-01-16 8518-11-16 8523-03-16 8523-03-16 8518-11-16 8518-11-16 8523-03-16
+9075-06-13 9073-04-13 9077-08-13 9077-08-13 9073-04-13 9073-04-13 9077-08-13
+9209-11-11 9207-09-11 9212-01-11 9212-01-11 9207-09-11 9207-09-11 9212-01-11
+9403-01-09 9400-11-09 9405-03-09 9405-03-09 9400-11-09 9400-11-09 9405-03-09
+PREHOOK: query: explain
+select
+ dateval,
+ dateval - date '1999-06-07',
+ date '1999-06-07' - dateval,
+ dateval - dateval
+from interval_arithmetic_1
+order by dateval
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select
+ dateval,
+ dateval - date '1999-06-07',
+ date '1999-06-07' - dateval,
+ dateval - dateval
+from interval_arithmetic_1
+order by dateval
+POSTHOOK: type: QUERY
+Explain
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: interval_arithmetic_1
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: dateval (type: date), (dateval - 1999-06-07) (type: interval_day_time), (1999-06-07 - dateval) (type: interval_day_time), (dateval - dateval) (type: interval_day_time)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: date)
+ sort order: +
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: interval_day_time), _col2 (type: interval_day_time), _col3 (type: interval_day_time)
+ Execution mode: vectorized
+ Reducer 2
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: date), VALUE._col0 (type: interval_day_time), VALUE._col1 (type: interval_day_time), VALUE._col2 (type: interval_day_time)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select
+ dateval,
+ dateval - date '1999-06-07',
+ date '1999-06-07' - dateval,
+ dateval - dateval
+from interval_arithmetic_1
+order by dateval
+PREHOOK: type: QUERY
+PREHOOK: Input: default@interval_arithmetic_1
+#### A masked pattern was here ####
+POSTHOOK: query: select
+ dateval,
+ dateval - date '1999-06-07',
+ date '1999-06-07' - dateval,
+ dateval - dateval
+from interval_arithmetic_1
+order by dateval
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@interval_arithmetic_1
+#### A masked pattern was here ####
+dateval c1 c2 c3
+0004-09-22 -728552 23:00:00.000000000 728552 23:00:00.000000000 0 00:00:00.000000000
+0528-10-27 -537126 23:00:00.000000000 537126 23:00:00.000000000 0 00:00:00.000000000
+1319-02-02 -248481 23:00:00.000000000 248481 23:00:00.000000000 0 00:00:00.000000000
+1404-07-23 -217263 23:00:00.000000000 217263 23:00:00.000000000 0 00:00:00.000000000
+1815-05-06 -67236 23:00:00.000000000 67236 23:00:00.000000000 0 00:00:00.000000000
+1883-04-17 -42418 23:00:00.000000000 42418 23:00:00.000000000 0 00:00:00.000000000
+1966-08-16 -11983 00:00:00.000000000 11983 00:00:00.000000000 0 00:00:00.000000000
+1973-04-17 -9546 23:00:00.000000000 9546 23:00:00.000000000 0 00:00:00.000000000
+1974-10-04 -9012 00:00:00.000000000 9012 00:00:00.000000000 0 00:00:00.000000000
+1976-03-03 -8495 23:00:00.000000000 8495 23:00:00.000000000 0 00:00:00.000000000
+1976-05-06 -8432 00:00:00.000000000 8432 00:00:00.000000000 0 00:00:00.000000000
+1978-08-05 -7611 00:00:00.000000000 7611 00:00:00.000000000 0 00:00:00.000000000
+1981-04-25 -6616 23:00:00.000000000 6616 23:00:00.000000000 0 00:00:00.000000000
+1981-11-15 -6412 23:00:00.000000000 6412 23:00:00.000000000 0 00:00:00.000000000
+1985-07-20 -5070 00:00:00.000000000 5070 00:00:00.000000000 0 00:00:00.000000000
+1985-11-18 -4948 23:00:00.000000000 4948 23:00:00.000000000 0 00:00:00.000000000
+1987-02-21 -4488 23:00:00.000000000 4488 23:00:00.000000000 0 00:00:00.000000000
+1987-05-28 -4393 00:00:00.000000000 4393 00:00:00.000000000 0 00:00:00.000000000
+1998-10-16 -234 00:00:00.000000000 234 00:00:00.000000000 0 00:00:00.000000000
+1999-10-03 118 00:00:00.000000000 -118 00:00:00.000000000 0 00:00:00.000000000
+2000-12-18 560 01:00:00.000000000 -560 01:00:00.000000000 0 00:00:00.000000000
+2002-05-10 1068 00:00:00.000000000 -1068 00:00:00.000000000 0 00:00:00.000000000
+2003-09-23 1569 00:00:00.000000000 -1569 00:00:00.000000000 0 00:00:00.000000000
+2004-03-07 1735 01:00:00.000000000 -1735 01:00:00.000000000 0 00:00:00.000000000
+2007-02-09 2804 01:00:00.000000000 -2804 01:00:00.000000000 0 00:00:00.000000000
+2009-01-21 3516 01:00:00.000000000 -3516 01:00:00.000000000 0 00:00:00.000000000
+2010-04-08 3958 00:00:00.000000000 -3958 00:00:00.000000000 0 00:00:00.000000000
+2013-04-07 5053 00:00:00.000000000 -5053 00:00:00.000000000 0 00:00:00.000000000
+2013-04-10 5056 00:00:00.000000000 -5056 00:00:00.000000000 0 00:00:00.000000000
+2021-09-24 8145 00:00:00.000000000 -8145 00:00:00.000000000 0 00:00:00.000000000
+2024-11-11 9289 01:00:00.000000000 -9289 01:00:00.000000000 0 00:00:00.000000000
+4143-07-08 783111 00:00:00.000000000 -783111 00:00:00.000000000 0 00:00:00.000000000
+4966-12-04 1083855 01:00:00.000000000 -1083855 01:00:00.000000000 0 00:00:00.000000000
+5339-02-01 1219784 01:00:00.000000000 -1219784 01:00:00.000000000 0 00:00:00.000000000
+5344-10-04 1221856 00:00:00.000000000 -1221856 00:00:00.000000000 0 00:00:00.000000000
+5397-07-13 1241131 00:00:00.000000000 -1241131 00:00:00.000000000 0 00:00:00.000000000
+5966-07-09 1448949 00:00:00.000000000 -1448949 00:00:00.000000000 0 00:00:00.000000000
+6229-06-28 1544997 00:00:00.000000000 -1544997 00:00:00.000000000 0 00:00:00.000000000
+6482-04-27 1637342 00:00:00.000000000 -1637342 00:00:00.000000000 0 00:00:00.000000000
+6631-11-13 1691962 01:00:00.000000000 -1691962 01:00:00.000000000 0 00:00:00.000000000
+6705-09-28 1718944 00:00:00.000000000 -1718944 00:00:00.000000000 0 00:00:00.000000000
+6731-02-12 1728212 01:00:00.000000000 -1728212 01:00:00.000000000 0 00:00:00.000000000
+7160-12-02 1885195 01:00:00.000000000 -1885195 01:00:00.000000000 0 00:00:00.000000000
+7409-09-07 1976054 00:00:00.000000000 -1976054 00:00:00.000000000 0 00:00:00.000000000
+7503-06-23 2010310 00:00:00.000000000 -2010310 00:00:00.000000000 0 00:00:00.000000000
+8422-07-22 2345998 00:00:00.000000000 -2345998 00:00:00.000000000 0 00:00:00.000000000
+8521-01-16 2381970 01:00:00.000000000 -2381970 01:00:00.000000000 0 00:00:00.000000000
+9075-06-13 2584462 00:00:00.000000000 -2584462 00:00:00.000000000 0 00:00:00.000000000
+9209-11-11 2633556 01:00:00.000000000 -2633556 01:00:00.000000000 0 00:00:00.000000000
+9403-01-09 2704106 01:00:00.000000000 -2704106 01:00:00.000000000 0 00:00:00.000000000
+PREHOOK: query: explain
+select
+ tsval,
+ tsval - interval '2-2' year to month,
+ tsval - interval '-2-2' year to month,
+ tsval + interval '2-2' year to month,
+ tsval + interval '-2-2' year to month,
+ - interval '2-2' year to month + tsval,
+ interval '2-2' year to month + tsval
+from interval_arithmetic_1
+order by tsval
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select
+ tsval,
+ tsval - interval '2-2' year to month,
+ tsval - interval '-2-2' year to month,
+ tsval + interval '2-2' year to month,
+ tsval + interval '-2-2' year to month,
+ - interval '2-2' year to month + tsval,
+ interval '2-2' year to month + tsval
+from interval_arithmetic_1
+order by tsval
+POSTHOOK: type: QUERY
+Explain
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: interval_arithmetic_1
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: tsval (type: timestamp), (tsval - 2-2) (type: timestamp), (tsval - -2-2) (type: timestamp), (tsval + 2-2) (type: timestamp), (tsval + -2-2) (type: timestamp), (-2-2 + tsval) (type: timestamp), (2-2 + tsval) (type: timestamp)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: timestamp)
+ sort order: +
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: timestamp), _col2 (type: timestamp), _col3 (type: timestamp), _col4 (type: timestamp), _col5 (type: timestamp), _col6 (type: timestamp)
+ Execution mode: vectorized
+ Reducer 2
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: timestamp), VALUE._col0 (type: timestamp), VALUE._col1 (type: timestamp), VALUE._col2 (type: timestamp), VALUE._col3 (type: timestamp), VALUE._col4 (type: timestamp), VALUE._col5 (type: timestamp)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select
+ tsval,
+ tsval - interval '2-2' year to month,
+ tsval - interval '-2-2' year to month,
+ tsval + interval '2-2' year to month,
+ tsval + interval '-2-2' year to month,
+ - interval '2-2' year to month + tsval,
+ interval '2-2' year to month + tsval
+from interval_arithmetic_1
+order by tsval
+PREHOOK: type: QUERY
+PREHOOK: Input: default@interval_arithmetic_1
+#### A masked pattern was here ####
+POSTHOOK: query: select
+ tsval,
+ tsval - interval '2-2' year to month,
+ tsval - interval '-2-2' year to month,
+ tsval + interval '2-2' year to month,
+ tsval + interval '-2-2' year to month,
+ - interval '2-2' year to month + tsval,
+ interval '2-2' year to month + tsval
+from interval_arithmetic_1
+order by tsval
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@interval_arithmetic_1
+#### A masked pattern was here ####
+tsval c1 c2 c3 c4 c5 c6
+0004-09-22 18:26:29.519542222 0002-07-22 18:26:29.519542222 0006-11-22 18:26:29.519542222 0006-11-22 18:26:29.519542222 0002-07-22 18:26:29.519542222 0002-07-22 18:26:29.519542222 0006-11-22 18:26:29.519542222
+0528-10-27 08:15:18.941718273 0526-08-27 08:15:18.941718273 0530-12-27 08:15:18.941718273 0530-12-27 08:15:18.941718273 0526-08-27 08:15:18.941718273 0526-08-27 08:15:18.941718273 0530-12-27 08:15:18.941718273
+1319-02-02 16:31:57.778 1316-12-02 16:31:57.778 1321-04-02 16:31:57.778 1321-04-02 16:31:57.778 1316-12-02 16:31:57.778 1316-12-02 16:31:57.778 1321-04-02 16:31:57.778
+1404-07-23 15:32:16.059185026 1402-05-23 15:32:16.059185026 1406-09-23 15:32:16.059185026 1406-09-23 15:32:16.059185026 1402-05-23 15:32:16.059185026 1402-05-23 15:32:16.059185026 1406-09-23 15:32:16.059185026
+1815-05-06 00:12:37.543584705 1813-03-06 00:12:37.543584705 1817-07-06 00:12:37.543584705 1817-07-06 00:12:37.543584705 1813-03-06 00:12:37.543584705 1813-03-06 00:12:37.543584705 1817-07-06 00:12:37.543584705
+1883-04-17 04:14:34.647766229 1881-02-17 04:14:34.647766229 1885-06-17 04:14:34.647766229 1885-06-17 04:14:34.647766229 1881-02-17 04:14:34.647766229 1881-02-17 04:14:34.647766229 1885-06-17 04:14:34.647766229
+1966-08-16 13:36:50.183618031 1964-06-16 13:36:50.183618031 1968-10-16 13:36:50.183618031 1968-10-16 13:36:50.183618031 1964-06-16 13:36:50.183618031 1964-06-16 13:36:50.183618031 1968-10-16 13:36:50.183618031
+1973-04-17 06:30:38.596784156 1971-02-17 06:30:38.596784156 1975-06-17 07:30:38.596784156 1975-06-17 07:30:38.596784156 1971-02-17 06:30:38.596784156 1971-02-17 06:30:38.596784156 1975-06-17 07:30:38.596784156
+1974-10-04 17:21:03.989 1972-08-04 17:21:03.989 1976-12-04 16:21:03.989 1976-12-04 16:21:03.989 1972-08-04 17:21:03.989 1972-08-04 17:21:03.989 1976-12-04 16:21:03.989
+1976-03-03 04:54:33.000895162 1974-01-03 04:54:33.000895162 1978-05-03 05:54:33.000895162 1978-05-03 05:54:33.000895162 1974-01-03 04:54:33.000895162 1974-01-03 04:54:33.000895162 1978-05-03 05:54:33.000895162
+1976-05-06 00:42:30.910786948 1974-03-06 00:42:30.910786948 1978-07-06 00:42:30.910786948 1978-07-06 00:42:30.910786948 1974-03-06 00:42:30.910786948 1974-03-06 00:42:30.910786948 1978-07-06 00:42:30.910786948
+1978-08-05 14:41:05.501 1976-06-05 14:41:05.501 1980-10-05 14:41:05.501 1980-10-05 14:41:05.501 1976-06-05 14:41:05.501 1976-06-05 14:41:05.501 1980-10-05 14:41:05.501
+1981-04-25 09:01:12.077192689 1979-02-25 09:01:12.077192689 1983-06-25 10:01:12.077192689 1983-06-25 10:01:12.077192689 1979-02-25 09:01:12.077192689 1979-02-25 09:01:12.077192689 1983-06-25 10:01:12.077192689
+1981-11-15 23:03:10.999338387 1979-09-16 00:03:10.999338387 1984-01-15 23:03:10.999338387 1984-01-15 23:03:10.999338387 1979-09-16 00:03:10.999338387 1979-09-16 00:03:10.999338387 1984-01-15 23:03:10.999338387
+1985-07-20 09:30:11 1983-05-20 09:30:11 1987-09-20 09:30:11 1987-09-20 09:30:11 1983-05-20 09:30:11 1983-05-20 09:30:11 1987-09-20 09:30:11
+1985-11-18 16:37:54 1983-09-18 17:37:54 1988-01-18 16:37:54 1988-01-18 16:37:54 1983-09-18 17:37:54 1983-09-18 17:37:54 1988-01-18 16:37:54
+1987-02-21 19:48:29 1984-12-21 19:48:29 1989-04-21 20:48:29 1989-04-21 20:48:29 1984-12-21 19:48:29 1984-12-21 19:48:29 1989-04-21 20:48:29
+1987-05-28 13:52:07.900916635 1985-03-28 12:52:07.900916635 1989-07-28 13:52:07.900916635 1989-07-28 13:52:07.900916635 1985-03-28 12:52:07.900916635 1985-03-28 12:52:07.900916635 1989-07-28 13:52:07.900916635
+1998-10-16 20:05:29.397591987 1996-08-16 20:05:29.397591987 2000-12-16 19:05:29.397591987 2000-12-16 19:05:29.397591987 1996-08-16 20:05:29.397591987 1996-08-16 20:05:29.397591987 2000-12-16 19:05:29.397591987
+1999-10-03 16:59:10.396903939 1997-08-03 16:59:10.396903939 2001-12-03 15:59:10.396903939 2001-12-03 15:59:10.396903939 1997-08-03 16:59:10.396903939 1997-08-03 16:59:10.396903939 2001-12-03 15:59:10.396903939
+2000-12-18 08:42:30.000595596 1998-10-18 09:42:30.000595596 2003-02-18 08:42:30.000595596 2003-02-18 08:42:30.000595596 1998-10-18 09:42:30.000595596 1998-10-18 09:42:30.000595596 2003-02-18 08:42:30.000595596
+2002-05-10 05:29:48.990818073 2000-03-10 04:29:48.990818073 2004-07-10 05:29:48.990818073 2004-07-10 05:29:48.990818073 2000-03-10 04:29:48.990818073 2000-03-10 04:29:48.990818073 2004-07-10 05:29:48.990818073
+2003-09-23 22:33:17.00003252 2001-07-23 22:33:17.00003252 2005-11-23 21:33:17.00003252 2005-11-23 21:33:17.00003252 2001-07-23 22:33:17.00003252 2001-07-23 22:33:17.00003252 2005-11-23 21:33:17.00003252
+2004-03-07 20:14:13 2002-01-07 20:14:13 2006-05-07 21:14:13 2006-05-07 21:14:13 2002-01-07 20:14:13 2002-01-07 20:14:13 2006-05-07 21:14:13
+2007-02-09 05:17:29.368756876 2004-12-09 05:17:29.368756876 2009-04-09 06:17:29.368756876 2009-04-09 06:17:29.368756876 2004-12-09 05:17:29.368756876 2004-12-09 05:17:29.368756876 2009-04-09 06:17:29.368756876
+2009-01-21 10:49:07.108 2006-11-21 10:49:07.108 2011-03-21 11:49:07.108 2011-03-21 11:49:07.108 2006-11-21 10:49:07.108 2006-11-21 10:49:07.108 2011-03-21 11:49:07.108
+2010-04-08 02:43:35.861742727 2008-02-08 01:43:35.861742727 2012-06-08 02:43:35.861742727 2012-06-08 02:43:35.861742727 2008-02-08 01:43:35.861742727 2008-02-08 01:43:35.861742727 2012-06-08 02:43:35.861742727
+2013-04-07 02:44:43.00086821 2011-02-07 01:44:43.00086821 2015-06-07 02:44:43.00086821 2015-06-07 02:44:43.00086821 2011-02-07 01:44:43.00086821 2011-02-07 01:44:43.00086821 2015-06-07 02:44:43.00086821
+2013-04-10 00:43:46.854731546 2011-02-09 23:43:46.854731546 2015-06-10 00:43:46.854731546 2015-06-10 00:43:46.854731546 2011-02-09 23:43:46.854731546 2011-02-09 23:43:46.854731546 2015-06-10 00:43:46.854731546
+2021-09-24 03:18:32.413655165 2019-07-24 03:18:32.413655165 2023-11-24 02:18:32.413655165 2023-11-24 02:18:32.413655165 2019-07-24 03:18:32.413655165 2019-07-24 03:18:32.413655165 2023-11-24 02:18:32.413655165
+2024-11-11 16:42:41.101 2022-09-11 17:42:41.101 2027-01-11 16:42:41.101 2027-01-11 16:42:41.101 2022-09-11 17:42:41.101 2022-09-11 17:42:41.101 2027-01-11 16:42:41.101
+4143-07-08 10:53:27.252802259 4141-05-08 10:53:27.252802259 4145-09-08 10:53:27.252802259 4145-09-08 10:53:27.252802259 4141-05-08 10:53:27.252802259 4141-05-08 10:53:27.252802259 4145-09-08 10:53:27.252802259
+4966-12-04 09:30:55.202 4964-10-04 10:30:55.202 4969-02-04 09:30:55.202 4969-02-04 09:30:55.202 4964-10-04 10:30:55.202 4964-10-04 10:30:55.202 4969-02-04 09:30:55.202
+5339-02-01 14:10:01.085678691 5336-12-01 14:10:01.085678691 5341-04-01 15:10:01.085678691 5341-04-01 15:10:01.085678691 5336-12-01 14:10:01.085678691 5336-12-01 14:10:01.085678691 5341-04-01 15:10:01.085678691
+5344-10-04 18:40:08.165 5342-08-04 18:40:08.165 5346-12-04 17:40:08.165 5346-12-04 17:40:08.165 5342-08-04 18:40:08.165 5342-08-04 18:40:08.165 5346-12-04 17:40:08.165
+5397-07-13 07:12:32.000896438 5395-05-13 07:12:32.000896438 5399-09-13 07:12:32.000896438 5399-09-13 07:12:32.000896438 5395-05-13 07:12:32.000896438 5395-05-13 07:12:32.000896438 5399-09-13 07:12:32.000896438
+5966-07-09 03:30:50.597 5964-05-09 03:30:50.597 5968-09-09 03:30:50.597 5968-09-09 03:30:50.597 5964-05-09 03:30:50.597 5964-05-09 03:30:50.597 5968-09-09 03:30:50.597
+6229-06-28 02:54:28.970117179 6227-04-28 02:54:28.970117179 6231-08-28 02:54:28.970117179 6231-08-28 02:54:28.970117179 6227-04-28 02:54:28.970117179 6227-04-28 02:54:28.970117179 6231-08-28 02:54:28.970117179
+6482-04-27 12:07:38.073915413 6480-02-27 11:07:38.073915413 6484-06-27 12:07:38.073915413 6484-06-27 12:07:38.073915413 6480-02-27 11:07:38.073915413 6480-02-27 11:07:38.073915413 6484-06-27 12:07:38.073915413
+6631-11-13 16:31:29.702202248 6629-09-13 17:31:29.702202248 6634-01-13 16:31:29.702202248 6634-01-13 16:31:29.702202248 6629-09-13 17:31:29.702202248 6629-09-13 17:31:29.702202248 6634-01-13 16:31:29.702202248
+6705-09-28 18:27:28.000845672 6703-07-28 18:27:28.000845672 6707-11-28 17:27:28.000845672 6707-11-28 17:27:28.000845672 6703-07-28 18:27:28.000845672 6703-07-28 18:27:28.000845672 6707-11-28 17:27:28.000845672
+6731-02-12 08:12:48.287783702 6728-12-12 08:12:48.287783702 6733-04-12 09:12:48.287783702 6733-04-12 09:12:48.287783702 6728-12-12 08:12:48.287783702 6728-12-12 08:12:48.287783702 6733-04-12 09:12:48.287783702
+7160-12-02 06:00:24.81200852 7158-10-02 07:00:24.81200852 7163-02-02 06:00:24.81200852 7163-02-02 06:00:24.81200852 7158-10-02 07:00:24.81200852 7158-10-02 07:00:24.81200852 7163-02-02 06:00:24.81200852
+7409-09-07 23:33:32.459349602 7407-07-07 23:33:32.459349602 7411-11-07 22:33:32.459349602 7411-11-07 22:33:32.459349602 7407-07-07 23:33:32.459349602 7407-07-07 23:33:32.459349602 7411-11-07 22:33:32.459349602
+7503-06-23 23:14:17.486 7501-04-23 23:14:17.486 7505-08-23 23:14:17.486 7505-08-23 23:14:17.486 7501-04-23 23:14:17.486 7501-04-23 23:14:17.486 7505-08-23 23:14:17.486
+8422-07-22 03:21:45.745036084 8420-05-22 03:21:45.745036084 8424-09-22 03:21:45.745036084 8424-09-22 03:21:45.745036084 8420-05-22 03:21:45.745036084 8420-05-22 03:21:45.745036084 8424-09-22 03:21:45.745036084
+8521-01-16 20:42:05.668832388 8518-11-16 20:42:05.668832388 8523-03-16 21:42:05.668832388 8523-03-16 21:42:05.668832388 8518-11-16 20:42:05.668832388 8518-11-16 20:42:05.668832388 8523-03-16 21:42:05.668832388
+9075-06-13 16:20:09.218517797 9073-04-13 16:20:09.218517797 9077-08-13 16:20:09.218517797 9077-08-13 16:20:09.218517797 9073-04-13 16:20:09.218517797 9073-04-13 16:20:09.218517797 9077-08-13 16:20:09.218517797
+9209-11-11 04:08:58.223768453 9207-09-11 05:08:58.223768453 9212-01-11 04:08:58.223768453 9212-01-11 04:08:58.223768453 9207-09-11 05:08:58.223768453 9207-09-11 05:08:58.223768453 9212-01-11 04:08:58.223768453
+9403-01-09 18:12:33.547 9400-11-09 18:12:33.547 9405-03-09 18:12:33.547 9405-03-09 18:12:33.547 9400-11-09 18:12:33.547 9400-11-09 18:12:33.547 9405-03-09 18:12:33.547
+PREHOOK: query: explain
+select
+ interval '2-2' year to month + interval '3-3' year to month,
+ interval '2-2' year to month - interval '3-3' year to month
+from interval_arithmetic_1
+order by interval '2-2' year to month + interval '3-3' year to month
+limit 2
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select
+ interval '2-2' year to month + interval '3-3' year to month,
+ interval '2-2' year to month - interval '3-3' year to month
+from interval_arithmetic_1
+order by interval '2-2' year to month + interval '3-3' year to month
+limit 2
+POSTHOOK: type: QUERY
+Explain
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: interval_arithmetic_1
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ Statistics: Num rows: 50 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: 5-5 (type: interval_year_month)
+ sort order: +
+ Statistics: Num rows: 50 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ TopN Hash Memory Usage: 0.1
+ Execution mode: vectorized
+ Reducer 2
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Select Operator
+ expressions: 5-5 (type: interval_year_month), -1-1 (type: interval_year_month)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 50 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Limit
+ Number of rows: 2
+ Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 2
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select
+ interval '2-2' year to month + interval '3-3' year to month,
+ interval '2-2' year to month - interval '3-3' year to month
+from interval_arithmetic_1
+order by interval '2-2' year to month + interval '3-3' year to month
+limit 2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@interval_arithmetic_1
+#### A masked pattern was here ####
+POSTHOOK: query: select
+ interval '2-2' year to month + interval '3-3' year to month,
+ interval '2-2' year to month - interval '3-3' year to month
+from interval_arithmetic_1
+order by interval '2-2' year to month + interval '3-3' year to month
+limit 2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@interval_arithmetic_1
+#### A masked pattern was here ####
+c0 c1
+5-5 -1-1
+5-5 -1-1
+PREHOOK: query: -- interval day-time arithmetic
+explain
+select
+ dateval,
+ dateval - interval '99 11:22:33.123456789' day to second,
+ dateval - interval '-99 11:22:33.123456789' day to second,
+ dateval + interval '99 11:22:33.123456789' day to second,
+ dateval + interval '-99 11:22:33.123456789' day to second,
+ -interval '99 11:22:33.123456789' day to second + dateval,
+ interval '99 11:22:33.123456789' day to second + dateval
+from interval_arithmetic_1
+order by dateval
+PREHOOK: type: QUERY
+POSTHOOK: query: -- interval day-time arithmetic
+explain
+select
+ dateval,
+ dateval - interval '99 11:22:33.123456789' day to second,
+ dateval - interval '-99 11:22:33.123456789' day to second,
+ dateval + interval '99 11:22:33.123456789' day to second,
+ dateval + interval '-99 11:22:33.123456789' day to second,
+ -interval '99 11:22:33.123456789' day to second + dateval,
+ interval '99 11:22:33.123456789' day to second + dateval
+from interval_arithmetic_1
+order by dateval
+POSTHOOK: type: QUERY
+Explain
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: interval_arithmetic_1
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: dateval (type: date), (dateval - 99 11:22:33.123456789) (type: timestamp), (dateval - -99 11:22:33.123456789) (type: timestamp), (dateval + 99 11:22:33.123456789) (type: timestamp), (dateval + -99 11:22:33.123456789) (type: timestamp), (-99 11:22:33.123456789 + dateval) (type: timestamp), (99 11:22:33.123456789 + dateval) (type: timestamp)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: date)
+ sort order: +
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: timestamp), _col2 (type: timestamp), _col3 (type: timestamp), _col4 (type: timestamp), _col5 (type: timestamp), _col6 (type: timestamp)
+ Execution mode: vectorized
+ Reducer 2
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: date), VALUE._col0 (type: timestamp), VALUE._col1 (type: timestamp), VALUE._col2 (type: timestamp), VALUE._col3 (type: timestamp), VALUE._col4 (type: timestamp), VALUE._col5 (type: timestamp)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select
+ dateval,
+ dateval - interval '99 11:22:33.123456789' day to second,
+ dateval - interval '-99 11:22:33.123456789' day to second,
+ dateval + interval '99 11:22:33.123456789' day to second,
+ dateval + interval '-99 11:22:33.123456789' day to second,
+ -interval '99 11:22:33.123456789' day to second + dateval,
+ interval '99 11:22:33.123456789' day to second + dateval
+from interval_arithmetic_1
+order by dateval
+PREHOOK: type: QUERY
+PREHOOK: Input: default@interval_arithmetic_1
+#### A masked pattern was here ####
+POSTHOOK: query: select
+ dateval,
+ dateval - interval '99 11:22:33.123456789' day to second,
+ dateval - interval '-99 11:22:33.123456789' day to second,
+ dateval + interval '99 11:22:33.123456789' day to second,
+ dateval + interval '-99 11:22:33.123456789' day to second,
+ -interval '99 11:22:33.123456789' day to second + dateval,
+ interval '99 11:22:33.123456789' day to second + dateval
+from interval_arithmetic_1
+order by dateval
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@interval_arithmetic_1
+#### A masked pattern was here ####
+dateval _c1 _c2 _c3 _c4 _c5 _c6
+0004-09-22 0004-06-14 12:37:26.876543211 0004-12-30 11:22:33.123456789 0004-12-30 11:22:33.123456789 0004-06-14 12:37:26.876543211 0004-06-14 12:37:26.876543211 0004-12-30 11:22:33.123456789
+0528-10-27 0528-07-19 12:37:26.876543211 0529-02-03 11:22:33.123456789 0529-02-03 11:22:33.123456789 0528-07-19 12:37:26.876543211 0528-07-19 12:37:26.876543211 0529-02-03 11:22:33.123456789
+1319-02-02 1318-10-25 12:37:26.876543211 1319-05-12 11:22:33.123456789 1319-05-12 11:22:33.123456789 1318-10-25 12:37:26.876543211 1318-10-25 12:37:26.876543211 1319-05-12 11:22:33.123456789
+1404-07-23 1404-04-14 12:37:26.876543211 1404-10-30 11:22:33.123456789 1404-10-30 11:22:33.123456789 1404-04-14 12:37:26.876543211 1404-04-14 12:37:26.876543211 1404-10-30 11:22:33.123456789
+1815-05-06 1815-01-26 12:37:26.876543211 1815-08-13 11:22:33.123456789 1815-08-13 11:22:33.123456789 1815-01-26 12:37:26.876543211 1815-01-26 12:37:26.876543211 1815-08-13 11:22:33.123456789
+1883-04-17 1883-01-07 12:37:26.876543211 1883-07-25 11:22:33.123456789 1883-07-25 11:22:33.123456789 1883-01-07 12:37:26.876543211 1883-01-07 12:37:26.876543211 1883-07-25 11:22:33.123456789
+1966-08-16 1966-05-08 12:37:26.876543211 1966-11-23 10:22:33.123456789 1966-11-23 10:22:33.123456789 1966-05-08 12:37:26.876543211 1966-05-08 12:37:26.876543211 1966-11-23 10:22:33.123456789
+1973-04-17 1973-01-07 12:37:26.876543211 1973-07-25 12:22:33.123456789 1973-07-25 12:22:33.123456789 1973-01-07 12:37:26.876543211 1973-01-07 12:37:26.876543211 1973-07-25 12:22:33.123456789
+1974-10-04 1974-06-26 12:37:26.876543211 1975-01-11 10:22:33.123456789 1975-01-11 10:22:33.123456789 1974-06-26 12:37:26.876543211 1974-06-26 12:37:26.876543211 1975-01-11 10:22:33.123456789
+1976-03-03 1975-11-24 12:37:26.876543211 1976-06-10 12:22:33.123456789 1976-06-10 12:22:33.123456789 1975-11-24 12:37:26.876543211 1975-11-24 12:37:26.876543211 1976-06-10 12:22:33.123456789
+1976-05-06 1976-01-27 11:37:26.876543211 1976-08-13 11:22:33.123456789 1976-08-13 11:22:33.123456789 1976-01-27 11:37:26.876543211 1976-01-27 11:37:26.876543211 1976-08-13 11:22:33.123456789
+1978-08-05 1978-04-27 11:37:26.876543211 1978-11-12 10:22:33.123456789 1978-11-12 10:22:33.123456789 1978-04-27 11:37:26.876543211 1978-04-27 11:37:26.876543211 1978-11-12 10:22:33.123456789
+1981-04-25 1981-01-15 12:37:26.876543211 1981-08-02 12:22:33.123456789 1981-08-02 12:22:33.123456789 1981-01-15 12:37:26.876543211 1981-01-15 12:37:26.876543211 1981-08-02 12:22:33.123456789
+1981-11-15 1981-08-07 13:37:26.876543211 1982-02-22 11:22:33.123456789 1982-02-22 11:22:33.123456789 1981-08-07 13:37:26.876543211 1981-08-07 13:37:26.876543211 1982-02-22 11:22:33.123456789
+1985-07-20 1985-04-11 11:37:26.876543211 1985-10-27 10:22:33.123456789 1985-10-27 10:22:33.123456789 1985-04-11 11:37:26.876543211 1985-04-11 11:37:26.876543211 1985-10-27 10:22:33.123456789
+1985-11-18 1985-08-10 13:37:26.876543211 1986-02-25 11:22:33.123456789 1986-02-25 11:22:33.123456789 1985-08-10 13:37:26.876543211 1985-08-10 13:37:26.876543211 1986-02-25 11:22:33.123456789
+1987-02-21 1986-11-13 12:37:26.876543211 1987-05-31 12:22:33.123456789 1987-05-31 12:22:33.123456789 1986-11-13 12:37:26.876543211 1986-11-13 12:37:26.876543211 1987-05-31 12:22:33.123456789
+1987-05-28 1987-02-17 11:37:26.876543211 1987-09-04 11:22:33.123456789 1987-09-04 11:22:33.123456789 1987-02-17 11:37:26.876543211 1987-02-17 11:37:26.876543211 1987-09-04 11:22:33.123456789
+1998-10-16 1998-07-08 12:37:26.876543211 1999-01-23 10:22:33.123456789 1999-01-23 10:22:33.123456789 1998-07-08 12:37:26.876543211 1998-07-08 12:37:26.876543211 1999-01-23 10:22:33.123456789
+1999-10-03 1999-06-25 12:37:26.876543211 2000-01-10 10:22:33.123456789 2000-01-10 10:22:33.123456789 1999-06-25 12:37:26.876543211 1999-06-25 12:37:26.876543211 2000-01-10 10:22:33.123456789
+2000-12-18 2000-09-09 13:37:26.876543211 2001-03-27 11:22:33.123456789 2001-03-27 11:22:33.123456789 2000-09-09 13:37:26.876543211 2000-09-09 13:37:26.876543211 2001-03-27 11:22:33.123456789
+2002-05-10 2002-01-30 11:37:26.876543211 2002-08-17 11:22:33.123456789 2002-08-17 11:22:33.123456789 2002-01-30 11:37:26.876543211 2002-01-30 11:37:26.876543211 2002-08-17 11:22:33.123456789
+2003-09-23 2003-06-15 12:37:26.876543211 2003-12-31 10:22:33.123456789 2003-12-31 10:22:33.123456789 2003-06-15 12:37:26.876543211 2003-06-15 12:37:26.876543211 2003-12-31 10:22:33.123456789
+2004-03-07 2003-11-28 12:37:26.876543211 2004-06-14 12:22:33.123456789 2004-06-14 12:22:33.123456789 2003-11-28 12:37:26.876543211 2003-11-28 12:37:26.876543211 2004-06-14 12:22:33.123456789
+2007-02-09 2006-11-01 12:37:26.876543211 2007-05-19 12:22:33.123456789 2007-05-19 12:22:33.123456789 2006-11-01 12:37:26.876543211 2006-11-01 12:37:26.876543211 2007-05-19 12:22:33.123456789
+2009-01-21 2008-10-13 13:37:26.876543211 2009-04-30 12:22:33.123456789 2009-04-30 12:22:33.123456789 2008-10-13 13:37:26.876543211 2008-10-13 13:37:26.876543211 2009-04-30 12:22:33.123456789
+2010-04-08 2009-12-29 11:37:26.876543211 2010-07-16 11:22:33.123456789 2010-07-16 11:22:33.123456789 2009-12-29 11:37:26.876543211 2009-12-29 11:37:26.876543211 2010-07-16 11:22:33.123456789
+2013-04-07 2012-12-28 11:37:26.876543211 2013-07-15 11:22:33.123456789 2013-07-15 11:22:33.123456789 2012-12-28 11:37:26.876543211 2012-12-28 11:37:26.876543211 2013-07-15 11:22:33.123456789
+2013-04-10 2012-12-31 11:37:26.876543211 2013-07-18 11:22:33.123456789 2013-07-18 11:22:33.123456789 2012-12-31 11:37:26.876543211 2012-12-31 11:37:26.876543211 2013-07-18 11:22:33.123456789
+2021-09-24 2021-06-16 12:37:26.876543211 2022-01-01 10:22:33.123456789 2022-01-01 10:22:33.123456789 2021-06-16 12:37:26.876543211 2021-06-16 12:37:26.876543211 2022-01-01 10:22:33.123456789
+2024-11-11 2024-08-03 13:37:26.876543211 2025-02-18 11:22:33.123456789 2025-02-18 11:22:33.123456789 2024-08-03 13:37:26.876543211 2024-08-03 13:37:26.876543211 2025-02-18 11:22:33.123456789
+4143-07-08 4143-03-30 12:37:26.876543211 4143-10-15 11:22:33.123456789 4143-10-15 11:22:33.123456789 4143-03-30 12:37:26.876543211 4143-03-30 12:37:26.876543211 4143-10-15 11:22:33.123456789
+4966-12-04 4966-08-26 13:37:26.876543211 4967-03-13 12:22:33.123456789 4967-03-13 12:22:33.123456789 4966-08-26 13:37:26.876543211 4966-08-26 13:37:26.876543211 4967-03-13 12:22:33.123456789
+5339-02-01 5338-10-24 13:37:26.876543211 5339-05-11 12:22:33.123456789 5339-05-11 12:22:33.123456789 5338-10-24 13:37:26.876543211 5338-10-24 13:37:26.876543211 5339-05-11 12:22:33.123456789
+5344-10-04 5344-06-26 12:37:26.876543211 5345-01-11 10:22:33.123456789 5345-01-11 10:22:33.123456789 5344-06-26 12:37:26.876543211 5344-06-26 12:37:26.876543211 5345-01-11 10:22:33.123456789
+5397-07-13 5397-04-04 12:37:26.876543211 5397-10-20 11:22:33.123456789 5397-10-20 11:22:33.123456789 5397-04-04 12:37:26.876543211 5397-04-04 12:37:26.876543211 5397-10-20 11:22:33.123456789
+5966-07-09 5966-03-31 12:37:26.876543211 5966-10-16 11:22:33.123456789 5966-10-16 11:22:33.123456789 5966-03-31 12:37:26.876543211 5966-03-31 12:37:26.876543211 5966-10-16 11:22:33.123456789
+6229-06-28 6229-03-20 12:37:26.876543211 6229-10-05 11:22:33.123456789 6229-10-05 11:22:33.123456789 6229-03-20 12:37:26.876543211 6229-03-20 12:37:26.876543211 6229-10-05 11:22:33.123456789
+6482-04-27 6482-01-17 11:37:26.876543211 6482-08-04 11:22:33.123456789 6482-08-04 11:22:33.123456789 6482-01-17 11:37:26.876543211 6482-01-17 11:37:26.876543211 6482-08-04 11:22:33.123456789
+6631-11-13 6631-08-05 13:37:26.876543211 6632-02-20 11:22:33.123456789 6632-02-20 11:22:33.123456789 6631-08-05 13:37:26.876543211 6631-08-05 13:37:26.876543211 6632-02-20 11:22:33.123456789
+6705-09-28 6705-06-20 12:37:26.876543211 6706-01-05 10:22:33.123456789 6706-01-05 10:22:33.123456789 6705-06-20 12:37:26.876543211 6705-06-20 12:37:26.876543211 6706-01-05 10:22:33.123456789
+6731-02-12 6730-11-04 12:37:26.876543211 6731-05-22 12:22:33.123456789 6731-05-22 12:22:33.123456789 6730-11-04 12:37:26.876543211 6730-11-04 12:37:26.876543211 6731-05-22 12:22:33.123456789
+7160-12-02 7160-08-24 13:37:26.876543211 7161-03-11 11:22:33.123456789 7161-03-11 11:22:33.123456789 7160-08-24 13:37:26.876543211 7160-08-24 13:37:26.876543211 7161-03-11 11:22:33.123456789
+7409-09-07 7409-05-30 12:37:26.876543211 7409-12-15 10:22:33.123456789 7409-12-15 10:22:33.123456789 7409-05-30 12:37:26.876543211 7409-05-30 12:37:26.876543211 7409-12-15 10:22:33.123456789
+7503-06-23 7503-03-15 12:37:26.876543211 7503-09-30 11:22:33.123456789 7503-09-30 11:22:33.123456789 7503-03-15 12:37:26.876543211 7503-03-15 12:37:26.876543211 7503-09-30 11:22:33.123456789
+8422-07-22 8422-04-13 12:37:26.876543211 8422-10-29 11:22:33.123456789 8422-10-29 11:22:33.123456789 8422-04-13 12:37:26.876543211 8422-04-13 12:37:26.876543211 8422-10-29 11:22:33.123456789
+8521-01-16 8520-10-08 13:37:26.876543211 8521-04-25 12:22:33.123456789 8521-04-25 12:22:33.123456789 8520-10-08 13:37:26.876543211 8520-10-08 13:37:26.876543211 8521-04-25 12:22:33.123456789
+9075-06-13 9075-03-05 11:37:26.876543211 9075-09-20 11:22:33.123456789 9075-09-20 11:22:33.123456789 9075-03-05 11:37:26.876543211 9075-03-05 11:37:26.876543211 9075-09-20 11:22:33.123456789
+9209-11-11 9209-08-03 13:37:26.876543211 9210-02-18 11:22:33.123456789 9210-02-18 11:22:33.123456789 9209-08-03 13:37:26.876543211 9209-08-03 13:37:26.876543211 9210-02-18 11:22:33.123456789
+9403-01-09 9402-10-01 13:37:26.876543211 9403-04-18 12:22:33.123456789 9403-04-18 12:22:33.123456789 9402-10-01 13:37:26.876543211 9402-10-01 13:37:26.876543211 9403-04-18 12:22:33.123456789
+PREHOOK: query: explain
+select
+ dateval,
+ tsval,
+ dateval - tsval,
+ tsval - dateval,
+ tsval - tsval
+from interval_arithmetic_1
+order by dateval
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select
+ dateval,
+ tsval,
+ dateval - tsval,
+ tsval - dateval,
+ tsval - tsval
+from interval_arithmetic_1
+order by dateval
+POSTHOOK: type: QUERY
+Explain
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: interval_arithmetic_1
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: dateval (type: date), tsval (type: timestamp), (dateval - tsval) (type: interval_day_time), (tsval - dateval) (type: interval_day_time), (tsval - tsval) (type: interval_day_time)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: date)
+ sort order: +
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: timestamp), _col2 (type: interval_day_time), _col3 (type: interval_day_time), _col4 (type: interval_day_time)
+ Execution mode: vectorized
+ Reducer 2
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: date), VALUE._col0 (type: timestamp), VALUE._col1 (type: interval_day_time), VALUE._col2 (type: interval_day_time), VALUE._col3 (type: interval_day_time)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select
+ dateval,
+ tsval,
+ dateval - tsval,
+ tsval - dateval,
+ tsval - tsval
+from interval_arithmetic_1
+order by dateval
+PREHOOK: type: QUERY
+PREHOOK: Input: default@interval_arithmetic_1
+#### A masked pattern was here ####
+POSTHOOK: query: select
+ dateval,
+ tsval,
+ dateval - tsval,
+ tsval - dateval,
+ tsval - tsval
+from interval_arithmetic_1
+order by dateval
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@interval_arithmetic_1
+#### A masked pattern was here ####
+dateval tsval c2 c3 c4
+0004-09-22 0004-09-22 18:26:29.519542222 -0 18:26:30.519542222 0 18:26:30.519542222 0 00:00:00.000000000
+0528-10-27 0528-10-27 08:15:18.941718273 -0 08:15:19.941718273 0 08:15:19.941718273 0 00:00:00.000000000
+1319-02-02 1319-02-02 16:31:57.778 -0 16:31:58.778000000 0 16:31:58.778000000 0 00:00:00.000000000
+1404-07-23 1404-07-23 15:32:16.059185026 -0 15:32:17.059185026 0 15:32:17.059185026 0 00:00:00.000000000
+1815-05-06 1815-05-06 00:12:37.543584705 -0 00:12:38.543584705 0 00:12:38.543584705 0 00:00:00.000000000
+1883-04-17 1883-04-17 04:14:34.647766229 -0 04:14:35.647766229 0 04:14:35.647766229 0 00:00:00.000000000
+1966-08-16 1966-08-16 13:36:50.183618031 -0 13:36:51.183618031 0 13:36:51.183618031 0 00:00:00.000000000
+1973-04-17 1973-04-17 06:30:38.596784156 -0 06:30:38.596784156 0 06:30:38.596784156 0 00:00:00.000000000
+1974-10-04 1974-10-04 17:21:03.989 -0 17:21:03.989000000 0 17:21:03.989000000 0 00:00:00.000000000
+1976-03-03 1976-03-03 04:54:33.000895162 -0 04:54:33.000895162 0 04:54:33.000895162 0 00:00:00.000000000
+1976-05-06 1976-05-06 00:42:30.910786948 -0 00:42:30.910786948 0 00:42:30.910786948 0 00:00:00.000000000
+1978-08-05 1978-08-05 14:41:05.501 -0 14:41:05.501000000 0 14:41:05.501000000 0 00:00:00.000000000
+1981-04-25 1981-04-25 09:01:12.077192689 -0 09:01:12.077192689 0 09:01:12.077192689 0 00:00:00.000000000
+1981-11-15 1981-11-15 23:03:10.999338387 -0 23:03:10.999338387 0 23:03:10.999338387 0 00:00:00.000000000
+1985-07-20 1985-07-20 09:30:11 -0 09:30:11.000000000 0 09:30:11.000000000 0 00:00:00.000000000
+1985-11-18 1985-11-18 16:37:54 -0 16:37:54.000000000 0 16:37:54.000000000 0 00:00:00.000000000
+1987-02-21 1987-02-21 19:48:29 -0 19:48:29.000000000 0 19:48:29.000000000 0 00:00:00.000000000
+1987-05-28 1987-05-28 13:52:07.900916635 -0 13:52:07.900916635 0 13:52:07.900916635 0 00:00:00.000000000
+1998-10-16 1998-10-16 20:05:29.397591987 -0 20:05:29.397591987 0 20:05:29.397591987 0 00:00:00.000000000
+1999-10-03 1999-10-03 16:59:10.396903939 -0 16:59:10.396903939 0 16:59:10.396903939 0 00:00:00.000000000
+2000-12-18 2000-12-18 08:42:30.000595596 -0 08:42:30.000595596 0 08:42:30.000595596 0 00:00:00.000000000
+2002-05-10 2002-05-10 05:29:48.990818073 -0 05:29:48.990818073 0 05:29:48.990818073 0 00:00:00.000000000
+2003-09-23 2003-09-23 22:33:17.00003252 -0 22:33:17.000032520 0 22:33:17.000032520 0 00:00:00.000000000
+2004-03-07 2004-03-07 20:14:13 -0 20:14:13.000000000 0 20:14:13.000000000 0 00:00:00.000000000
+2007-02-09 2007-02-09 05:17:29.368756876 -0 05:17:29.368756876 0 05:17:29.368756876 0 00:00:00.000000000
+2009-01-21 2009-01-21 10:49:07.108 -0 10:49:07.108000000 0 10:49:07.108000000 0 00:00:00.000000000
+2010-04-08 2010-04-08 02:43:35.861742727 -0 02:43:35.861742727 0 02:43:35.861742727 0 00:00:00.000000000
+2013-04-07 2013-04-07 02:44:43.00086821 -0 02:44:43.000868210 0 02:44:43.000868210 0 00:00:00.000000000
+2013-04-10 2013-04-10 00:43:46.854731546 -0 00:43:46.854731546 0 00:43:46.854731546 0 00:00:00.000000000
+2021-09-24 2021-09-24 03:18:32.413655165 -0 03:18:32.413655165 0 03:18:32.413655165 0 00:00:00.000000000
+2024-11-11 2024-11-11 16:42:41.101 -0 16:42:41.101000000 0 16:42:41.101000000 0 00:00:00.000000000
+4143-07-08 4143-07-08 10:53:27.252802259 -0 10:53:27.252802259 0 10:53:27.252802259 0 00:00:00.000000000
+4966-12-04 4966-12-04 09:30:55.202 -0 09:30:55.202000000 0 09:30:55.202000000 0 00:00:00.000000000
+5339-02-01 5339-02-01 14:10:01.085678691 -0 14:10:01.085678691 0 14:10:01.085678691 0 00:00:00.000000000
+5344-10-04 5344-10-04 18:40:08.165 -0 18:40:08.165000000 0 18:40:08.165000000 0 00:00:00.000000000
+5397-07-13 5397-07-13 07:12:32.000896438 -0 07:12:32.000896438 0 07:12:32.000896438 0 00:00:00.000000000
+5966-07-09 5966-07-09 03:30:50.597 -0 03:30:50.597000000 0 03:30:50.597000000 0 00:00:00.000000000
+6229-06-28 6229-06-28 02:54:28.970117179 -0 02:54:28.970117179 0 02:54:28.970117179 0 00:00:00.000000000
+6482-04-27 6482-04-27 12:07:38.073915413 -0 12:07:38.073915413 0 12:07:38.073915413 0 00:00:00.000000000
+6631-11-13 6631-11-13 16:31:29.702202248 -0 16:31:29.702202248 0 16:31:29.702202248 0 00:00:00.000000000
+6705-09-28 6705-09-28 18:27:28.000845672 -0 18:27:28.000845672 0 18:27:28.000845672 0 00:00:00.000000000
+6731-02-12 6731-02-12 08:12:48.287783702 -0 08:12:48.287783702 0 08:12:48.287783702 0 00:00:00.000000000
+7160-12-02 7160-12-02 06:00:24.81200852 -0 06:00:24.812008520 0 06:00:24.812008520 0 00:00:00.000000000
+7409-09-07 7409-09-07 23:33:32.459349602 -0 23:33:32.459349602 0 23:33:32.459349602 0 00:00:00.000000000
+7503-06-23 7503-06-23 23:14:17.486 -0 23:14:17.486000000 0 23:14:17.486000000 0 00:00:00.000000000
+8422-07-22 8422-07-22 03:21:45.745036084 -0 03:21:45.745036084 0 03:21:45.745036084 0 00:00:00.000000000
+8521-01-16 8521-01-16 20:42:05.668832388 -0 20:42:05.668832388 0 20:42:05.668832388 0 00:00:00.000000000
+9075-06-13 9075-06-13 16:20:09.218517797 -0 16:20:09.218517797 0 16:20:09.218517797 0 00:00:00.000000000
+9209-11-11 9209-11-11 04:08:58.223768453 -0 04:08:58.223768453 0 04:08:58.223768453 0 00:00:00.000000000
+9403-01-09 9403-01-09 18:12:33.547 -0 18:12:33.547000000 0 18:12:33.547000000 0 00:00:00.000000000
+PREHOOK: query: explain
+select
+ tsval,
+ tsval - interval '99 11:22:33.123456789' day to second,
+ tsval - interval '-99 11:22:33.123456789' day to second,
+ tsval + interval '99 11:22:33.123456789' day to second,
+ tsval + interval '-99 11:22:33.123456789' day to second,
+ -interval '99 11:22:33.123456789' day to second + tsval,
+ interval '99 11:22:33.123456789' day to second + tsval
+from interval_arithmetic_1
+order by tsval
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select
+ tsval,
+ tsval - interval '99 11:22:33.123456789' day to second,
+ tsval - interval '-99 11:22:33.123456789' day to second,
+ tsval + interval '99 11:22:33.123456789' day to second,
+ tsval + interval '-99 11:22:33.123456789' day to second,
+ -interval '99 11:22:33.123456789' day to second + tsval,
+ interval '99 11:22:33.123456789' day to second + tsval
+from interval_arithmetic_1
+order by tsval
+POSTHOOK: type: QUERY
+Explain
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: interval_arithmetic_1
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: tsval (type: timestamp), (tsval - 99 11:22:33.123456789) (type: timestamp), (tsval - -99 11:22:33.123456789) (type: timestamp), (tsval + 99 11:22:33.123456789) (type: timestamp), (tsval + -99 11:22:33.123456789) (type: timestamp), (-99 11:22:33.123456789 + tsval) (type: timestamp), (99 11:22:33.123456789 + tsval) (type: timestamp)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: timestamp)
+ sort order: +
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: timestamp), _col2 (type: timestamp), _col3 (type: timestamp), _col4 (type: timestamp), _col5 (type: timestamp), _col6 (type: timestamp)
+ Execution mode: vectorized
+ Reducer 2
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: timestamp), VALUE._col0 (type: timestamp), VALUE._col1 (type: timestamp), VALUE._col2 (type: timestamp), VALUE._col3 (type: timestamp), VALUE._col4 (type: timestamp), VALUE._col5 (type: timestamp)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select
+ tsval,
+ tsval - interval '99 11:22:33.123456789' day to second,
+ tsval - interval '-99 11:22:33.123456789' day to second,
+ tsval + interval '99 11:22:33.123456789' day to second,
+ tsval + interval '-99 11:22:33.123456789' day to second,
+ -interval '99 11:22:33.123456789' day to second + tsval,
+ interval '99 11:22:33.123456789' day to second + tsval
+from interval_arithmetic_1
+order by tsval
+PREHOOK: type: QUERY
+PREHOOK: Input: default@interval_arithmetic_1
+#### A masked pattern was here ####
+POSTHOOK: query: select
+ tsval,
+ tsval - interval '99 11:22:33.123456789' day to second,
+ tsval - interval '-99 11:22:33.123456789' day to second,
+ tsval + interval '99 11:22:33.123456789' day to second,
+ tsval + interval '-99 11:22:33.123456789' day to second,
+ -interval '99 11:22:33.123456789' day to second + tsval,
+ interval '99 11:22:33.123456789' day to second + tsval
+from interval_arithmetic_1
+order by tsval
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@interval_arithmetic_1
+#### A masked pattern was here ####
+tsval _c1 _c2 _c3 _c4 _c5 _c6
+0004-09-22 18:26:29.519542222 0004-06-15 07:03:56.396085433 0004-12-31 05:49:02.642999011 0004-12-31 05:49:02.642999011 0004-06-15 07:03:56.396085433 0004-06-15 07:03:56.396085433 0004-12-31 05:49:02.642999011
+0528-10-27 08:15:18.941718273 0528-07-19 20:52:45.818261484 0529-02-03 19:37:52.065175062 0529-02-03 19:37:52.065175062 0528-07-19 20:52:45.818261484 0528-07-19 20:52:45.818261484 0529-02-03 19:37:52.065175062
+1319-02-02 16:31:57.778 1318-10-26 05:09:24.654543211 1319-05-13 03:54:30.901456789 1319-05-13 03:54:30.901456789 1318-10-26 05:09:24.654543211 1318-10-26 05:09:24.654543211 1319-05-13 03:54:30.901456789
+1404-07-23 15:32:16.059185026 1404-04-15 04:09:42.935728237 1404-10-31 02:54:49.182641815 1404-10-31 02:54:49.182641815 1404-04-15 04:09:42.935728237 1404-04-15 04:09:42.935728237 1404-10-31 02:54:49.182641815
+1815-05-06 00:12:37.543584705 1815-01-26 12:50:04.420127916 1815-08-13 11:35:10.667041494 1815-08-13 11:35:10.667041494 1815-01-26 12:50:04.420127916 1815-01-26 12:50:04.420127916 1815-08-13 11:35:10.667041494
+1883-04-17 04:14:34.647766229 1883-01-07 16:52:01.52430944 1883-07-25 15:37:07.771223018 1883-07-25 15:37:07.771223018 1883-01-07 16:52:01.52430944 1883-01-07 16:52:01.52430944 1883-07-25 15:37:07.771223018
+1966-08-16 13:36:50.183618031 1966-05-09 02:14:17.060161242 1966-11-23 23:59:23.30707482 1966-11-23 23:59:23.30707482 1966-05-09 02:14:17.060161242 1966-05-09 02:14:17.060161242 1966-11-23 23:59:23.30707482
+1973-04-17 06:30:38.596784156 1973-01-07 19:08:05.473327367 1973-07-25 18:53:11.720240945 1973-07-25 18:53:11.720240945 1973-01-07 19:08:05.473327367 1973-01-07 19:08:05.473327367 1973-07-25 18:53:11.720240945
+1974-10-04 17:21:03.989 1974-06-27 05:58:30.865543211 1975-01-12 03:43:37.112456789 1975-01-12 03:43:37.112456789 1974-06-27 05:58:30.865543211 1974-06-27 05:58:30.865543211 1975-01-12 03:43:37.112456789
+1976-03-03 04:54:33.000895162 1975-11-24 17:31:59.877438373 1976-06-10 17:17:06.124351951 1976-06-10 17:17:06.124351951 1975-11-24 17:31:59.877438373 1975-11-24 17:31:59.877438373 1976-06-10 17:17:06.124351951
+1976-05-06 00:42:30.910786948 1976-01-27 12:19:57.787330159 1976-08-13 12:05:04.034243737 1976-08-13 12:05:04.034243737 1976-01-27 12:19:57.787330159 1976-01-27 12:19:57.787330159 1976-08-13 12:05:04.034243737
+1978-08-05 14:41:05.501 1978-04-28 02:18:32.377543211 1978-11-13 01:03:38.624456789 1978-11-13 01:03:38.624456789 1978-04-28 02:18:32.377543211 1978-04-28 02:18:32.377543211 1978-11-13 01:03:38.624456789
+1981-04-25 09:01:12.077192689 1981-01-15 21:38:38.9537359 1981-08-02 21:23:45.200649478 1981-08-02 21:23:45.200649478 1981-01-15 21:38:38.9537359 1981-01-15 21:38:38.9537359 1981-08-02 21:23:45.200649478
+1981-11-15 23:03:10.999338387 1981-08-08 12:40:37.875881598 1982-02-23 10:25:44.122795176 1982-02-23 10:25:44.122795176 1981-08-08 12:40:37.875881598 1981-08-08 12:40:37.875881598 1982-02-23 10:25:44.122795176
+1985-07-20 09:30:11 1985-04-11 21:07:37.876543211 1985-10-27 19:52:44.123456789 1985-10-27 19:52:44.123456789 1985-04-11 21:07:37.876543211 1985-04-11 21:07:37.876543211 1985-10-27 19:52:44.123456789
+1985-11-18 16:37:54 1985-08-11 06:15:20.876543211 1986-02-26 04:00:27.123456789 1986-02-26 04:00:27.123456789 1985-08-11 06:15:20.876543211 1985-08-11 06:15:20.876543211 1986-02-26 04:00:27.123456789
+1987-02-21 19:48:29 1986-11-14 08:25:55.876543211 1987-06-01 08:11:02.123456789 1987-06-01 08:11:02.123456789 1986-11-14 08:25:55.876543211 1986-11-14 08:25:55.876543211 1987-06-01 08:11:02.123456789
+1987-05-28 13:52:07.900916635 1987-02-18 01:29:34.777459846 1987-09-05 01:14:41.024373424 1987-09-05 01:14:41.024373424 1987-02-18 01:29:34.777459846 1987-02-18 01:29:34.777459846 1987-09-05 01:14:41.024373424
+1998-10-16 20:05:29.397591987 1998-07-09 08:42:56.274135198 1999-01-24 06:28:02.521048776 1999-01-24 06:28:02.521048776 1998-07-09 08:42:56.274135198 1998-07-09 08:42:56.274135198 1999-01-24 06:28:02.521048776
+1999-10-03 16:59:10.396903939 1999-06-26 05:36:37.27344715 2000-01-11 03:21:43.520360728 2000-01-11 03:21:43.520360728 1999-06-26 05:36:37.27344715 1999-06-26 05:36:37.27344715 2000-01-11 03:21:43.520360728
+2000-12-18 08:42:30.000595596 2000-09-09 22:19:56.877138807 2001-03-27 20:05:03.124052385 2001-03-27 20:05:03.124052385 2000-09-09 22:19:56.877138807 2000-09-09 22:19:56.877138807 2001-03-27 20:05:03.124052385
+2002-05-10 05:29:48.990818073 2002-01-30 17:07:15.867361284 2002-08-17 16:52:22.114274862 2002-08-17 16:52:22.114274862 2002-01-30 17:07:15.867361284 2002-01-30 17:07:15.867361284 2002-08-17 16:52:22.114274862
+2003-09-23 22:33:17.00003252 2003-06-16 11:10:43.876575731 2004-01-01 08:55:50.123489309 2004-01-01 08:55:50.123489309 2003-06-16 11:10:43.876575731 2003-06-16 11:10:43.876575731 2004-01-01 08:55:50.123489309
+2004-03-07 20:14:13 2003-11-29 08:51:39.876543211 2004-06-15 08:36:46.123456789 2004-06-15 08:36:46.123456789 2003-11-29 08:51:39.876543211 2003-11-29 08:51:39.876543211 2004-06-15 08:36:46.123456789
+2007-02-09 05:17:29.368756876 2006-11-01 17:54:56.245300087 2007-05-19 17:40:02.492213665 2007-05-19 17:40:02.492213665 2006-11-01 17:54:56.245300087 2006-11-01 17:54:56.245300087 2007-05-19 17:40:02.492213665
+2009-01-21 10:49:07.108 2008-10-14 00:26:33.984543211 2009-04-30 23:11:40.231456789 2009-04-30 23:11:40.231456789 2008-10-14 00:26:33.984543211 2008-10-14 00:26:33.984543211 2009-04-30 23:11:40.231456789
+2010-04-08 02:43:35.861742727 2009-12-29 14:21:02.738285938 2010-07-16 14:06:08.985199516 2010-07-16 14:06:08.985199516 2009-12-29 14:21:02.738285938 2009-12-29 14:21:02.738285938 2010-07-16 14:06:08.985199516
+2013-04-07 02:44:43.00086821 2012-12-28 14:22:09.877411421 2013-07-15 14:07:16.124324999 2013-07-15 14:07:16.124324999 2012-12-28 14:22:09.877411421 2012-12-28 14:22:09.877411421 2013-07-15 14:07:16.124324999
+2013-04-10 00:43:46.854731546 2012-12-31 12:21:13.731274757 2013-07-18 12:06:19.978188335 2013-07-18 12:06:19.978188335 2012-12-31 12:21:13.731274757 2012-12-31 12:21:13.731274757 2013-07-18 12:06:19.978188335
+2021-09-24 03:18:32.413655165 2021-06-16 15:55:59.290198376 2022-01-01 13:41:05.537111954 2022-01-01 13:41:05.537111954 2021-06-16 15:55:59.290198376 2021-06-16 15:55:59.290198376 2022-01-01 13:41:05.537111954
+2024-11-11 16:42:41.101 2024-08-04 06:20:07.977543211 2025-02-19 04:05:14.224456789 2025-02-19 04:05:14.224456789 2024-08-04 06:20:07.977543211 2024-08-04 06:20:07.977543211 2025-02-19 04:05:14.224456789
+4143-07-08 10:53:27.252802259 4143-03-30 23:30:54.12934547 4143-10-15 22:16:00.376259048 4143-10-15 22:16:00.376259048 4143-03-30 23:30:54.12934547 4143-03-30 23:30:54.12934547 4143-10-15 22:16:00.376259048
+4966-12-04 09:30:55.202 4966-08-26 23:08:22.078543211 4967-03-13 21:53:28.325456789 4967-03-13 21:53:28.325456789 4966-08-26 23:08:22.078543211 4966-08-26 23:08:22.078543211 4967-03-13 21:53:28.325456789
+5339-02-01 14:10:01.085678691 5338-10-25 03:47:27.962221902 5339-05-12 02:32:34.20913548 5339-05-12 02:32:34.20913548 5338-10-25 03:47:27.962221902 5338-10-25 03:47:27.962221902 5339-05-12 02:32:34.20913548
+5344-10-04 18:40:08.165 5344-06-27 07:17:35.041543211 5345-01-12 05:02:41.288456789 5345-01-12 05:02:41.288456789 5344-06-27 07:17:35.041543211 5344-06-27 07:17:35.041543211 5345-01-12 05:02:41.288456789
+5397-07-13 07:12:32.000896438 5397-04-04 19:49:58.877439649 5397-10-20 18:35:05.124353227 5397-10-20 18:35:05.124353227 5397-04-04 19:49:58.877439649 5397-04-04 19:49:58.877439649 5397-10-20 18:35:05.124353227
+5966-07-09 03:30:50.597 5966-03-31 16:08:17.473543211 5966-10-16 14:53:23.720456789 5966-10-16 14:53:23.720456789 5966-03-31 16:08:17.473543211 5966-03-31 16:08:17.473543211 5966-10-16 14:53:23.720456789
+6229-06-28 02:54:28.970117179 6229-03-20 15:31:55.84666039 6229-10-05 14:17:02.093573968 6229-10-05 14:17:02.093573968 6229-03-20 15:31:55.84666039 6229-03-20 15:31:55.84666039 6229-10-05 14:17:02.093573968
+6482-04-27 12:07:38.073915413 6482-01-17 23:45:04.950458624 6482-08-04 23:30:11.197372202 6482-08-04 23:30:11.197372202 6482-01-17 23:45:04.950458624 6482-01-17 23:45:04.950458624 6482-08-04 23:30:11.197372202
+6631-11-13 16:31:29.702202248 6631-08-06 06:08:56.578745459 6632-02-21 03:54:02.825659037 6632-02-21 03:54:02.825659037 6631-08-06 06:08:56.578745459 6631-08-06 06:08:56.578745459 6632-02-21 03:54:02.825659037
+6705-09-28 18:27:28.000845672 6705-06-21 07:04:54.877388883 6706-01-06 04:50:01.124302461 6706-01-06 04:50:01.124302461 6705-06-21 07:04:54.877388883 6705-06-21 07:04:54.877388883 6706-01-06 04:50:01.124302461
+6731-02-12 08:12:48.287783702 6730-11-04 20:50:15.164326913 6731-05-22 20:35:21.411240491 6731-05-22 20:35:21.411240491 6730-11-04 20:50:15.164326913 6730-11-04 20:50:15.164326913 6731-05-22 20:35:21.411240491
+7160-12-02 06:00:24.81200852 7160-08-24 19:37:51.688551731 7161-03-11 17:22:57.935465309 7161-03-11 17:22:57.935465309 7160-08-24 19:37:51.688551731 7160-08-24 19:37:51.688551731 7161-03-11 17:22:57.935465309
+7409-09-07 23:33:32.459349602 7409-05-31 12:10:59.335892813 7409-12-16 09:56:05.582806391 7409-12-16 09:56:05.582806391 7409-05-31 12:10:59.335892813 7409-05-31 12:10:59.335892813 7409-12-16 09:56:05.582806391
+7503-06-23 23:14:17.486 7503-03-16 11:51:44.362543211 7503-10-01 10:36:50.609456789 7503-10-01 10:36:50.609456789 7503-03-16 11:51:44.362543211 7503-03-16 11:51:44.362543211 7503-10-01 10:36:50.609456789
+8422-07-22 03:21:45.745036084 8422-04-13 15:59:12.621579295 8422-10-29 14:44:18.868492873 8422-10-29 14:44:18.868492873 8422-04-13 15:59:12.621579295 8422-04-13 15:59:12.621579295 8422-10-29 14:44:18.868492873
+8521-01-16 20:42:05.668832388 8520-10-09 10:19:32.545375599 8521-04-26 09:04:38.792289177 8521-04-26 09:04:38.792289177 8520-10-09 10:19:32.545375599 8520-10-09 10:19:32.545375599 8521-04-26 09:04:38.792289177
+9075-06-13 16:20:09.218517797 9075-03-06 03:57:36.095061008 9075-09-21 03:42:42.341974586 9075-09-21 03:42:42.341974586 9075-03-06 03:57:36.095061008 9075-03-06 03:57:36.095061008 9075-09-21 03:42:42.341974586
+9209-11-11 04:08:58.223768453 9209-08-03 17:46:25.100311664 9210-02-18 15:31:31.347225242 9210-02-18 15:31:31.347225242 9209-08-03 17:46:25.100311664 9209-08-03 17:46:25.100311664 9210-02-18 15:31:31.347225242
+9403-01-09 18:12:33.547 9402-10-02 07:50:00.423543211 9403-04-19 06:35:06.670456789 9403-04-19 06:35:06.670456789 9402-10-02 07:50:00.423543211 9402-10-02 07:50:00.423543211 9403-04-19 06:35:06.670456789
+PREHOOK: query: explain
+select
+ interval '99 11:22:33.123456789' day to second + interval '10 9:8:7.123456789' day to second,
+ interval '99 11:22:33.123456789' day to second - interval '10 9:8:7.123456789' day to second
+from interval_arithmetic_1
+limit 2
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select
+ interval '99 11:22:33.123456789' day to second + interval '10 9:8:7.123456789' day to second,
+ interval '99 11:22:33.123456789' day to second - interval '10 9:8:7.123456789' day to second
+from interval_arithmetic_1
+limit 2
+POSTHOOK: type: QUERY
+Explain
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: interval_arithmetic_1
+ Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: 109 20:30:40.246913578 (type: interval_day_time), 89 02:14:26.000000000 (type: interval_day_time)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 50 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Limit
+ Number of rows: 2
+ Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 2
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select
+ interval '99 11:22:33.123456789' day to second + interval '10 9:8:7.123456789' day to second,
+ interval '99 11:22:33.123456789' day to second - interval '10 9:8:7.123456789' day to second
+from interval_arithmetic_1
+limit 2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@interval_arithmetic_1
+#### A masked pattern was here ####
+POSTHOOK: query: select
+ interval '99 11:22:33.123456789' day to second + interval '10 9:8:7.123456789' day to second,
+ interval '99 11:22:33.123456789' day to second - interval '10 9:8:7.123456789' day to second
+from interval_arithmetic_1
+limit 2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@interval_arithmetic_1
+#### A masked pattern was here ####
+_c0 _c1
+109 20:30:40.246913578 89 02:14:26.000000000
+109 20:30:40.246913578 89 02:14:26.000000000
+PREHOOK: query: drop table interval_arithmetic_1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@interval_arithmetic_1
+PREHOOK: Output: default@interval_arithmetic_1
+POSTHOOK: query: drop table interval_arithmetic_1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@interval_arithmetic_1
+POSTHOOK: Output: default@interval_arithmetic_1
[30/50] [abbrv] hive git commit: HIVE-13111: Fix timestamp /
interval_day_time wrong results with HIVE-9862 (Matt McCline,
reviewed by Jason Dere)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDoubleToTimestamp.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDoubleToTimestamp.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDoubleToTimestamp.java
index 39823fe..31d2f78 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDoubleToTimestamp.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDoubleToTimestamp.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.*;
+import org.apache.hadoop.hive.serde2.io.TimestampWritable;
public class CastDoubleToTimestamp extends VectorExpression {
private static final long serialVersionUID = 1L;
@@ -37,9 +38,11 @@ public class CastDoubleToTimestamp extends VectorExpression {
super();
}
- private void setSecondsWithFractionalNanoseconds(TimestampColumnVector timestampColVector,
+ private void setDouble(TimestampColumnVector timestampColVector,
double[] vector, int elementNum) {
- timestampColVector.setTimestampSecondsWithFractionalNanoseconds(elementNum, vector[elementNum]);
+ TimestampWritable.setTimestampFromDouble(
+ timestampColVector.getScratchTimestamp(), vector[elementNum]);
+ timestampColVector.setFromScratchTimestamp(elementNum);
}
@Override
@@ -66,7 +69,7 @@ public class CastDoubleToTimestamp extends VectorExpression {
if (inputColVector.isRepeating) {
//All must be selected otherwise size would be zero
//Repeating property will not change.
- setSecondsWithFractionalNanoseconds(outputColVector, vector, 0);
+ setDouble(outputColVector, vector, 0);
// Even if there are no nulls, we always copy over entry 0. Simplifies code.
outputIsNull[0] = inputIsNull[0];
outputColVector.isRepeating = true;
@@ -74,11 +77,11 @@ public class CastDoubleToTimestamp extends VectorExpression {
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- setSecondsWithFractionalNanoseconds(outputColVector, vector, i);
+ setDouble(outputColVector, vector, i);
}
} else {
for(int i = 0; i != n; i++) {
- setSecondsWithFractionalNanoseconds(outputColVector, vector, i);
+ setDouble(outputColVector, vector, i);
}
}
outputColVector.isRepeating = false;
@@ -86,12 +89,12 @@ public class CastDoubleToTimestamp extends VectorExpression {
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- setSecondsWithFractionalNanoseconds(outputColVector, vector, i);
+ setDouble(outputColVector, vector, i);
outputIsNull[i] = inputIsNull[i];
}
} else {
for(int i = 0; i != n; i++) {
- setSecondsWithFractionalNanoseconds(outputColVector, vector, i);
+ setDouble(outputColVector, vector, i);
}
System.arraycopy(inputIsNull, 0, outputIsNull, 0, n);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastLongToTimestamp.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastLongToTimestamp.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastLongToTimestamp.java
index d344d4d..a2ee52d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastLongToTimestamp.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastLongToTimestamp.java
@@ -20,8 +20,7 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.*;
-import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
-import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
+import org.apache.hadoop.hive.serde2.io.TimestampWritable;
public class CastLongToTimestamp extends VectorExpression {
private static final long serialVersionUID = 1L;
@@ -40,7 +39,10 @@ public class CastLongToTimestamp extends VectorExpression {
}
private void setSeconds(TimestampColumnVector timestampColVector, long[] vector, int elementNum) {
- timestampColVector.setTimestampSeconds(elementNum, vector[elementNum]);
+ TimestampWritable.setTimestampFromLong(
+ timestampColVector.getScratchTimestamp(), vector[elementNum],
+ /* intToTimestampInSeconds */ true);
+ timestampColVector.setFromScratchTimestamp(elementNum);
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastMillisecondsLongToTimestamp.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastMillisecondsLongToTimestamp.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastMillisecondsLongToTimestamp.java
index a0c947f..01c8810 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastMillisecondsLongToTimestamp.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastMillisecondsLongToTimestamp.java
@@ -20,8 +20,7 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.*;
-import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
-import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
+import org.apache.hadoop.hive.serde2.io.TimestampWritable;
public class CastMillisecondsLongToTimestamp extends VectorExpression {
private static final long serialVersionUID = 1L;
@@ -39,6 +38,13 @@ public class CastMillisecondsLongToTimestamp extends VectorExpression {
super();
}
+ private void setMilliseconds(TimestampColumnVector timestampColVector, long[] vector, int elementNum) {
+ TimestampWritable.setTimestampFromLong(
+ timestampColVector.getScratchTimestamp(), vector[elementNum],
+ /* intToTimestampInSeconds */ false);
+ timestampColVector.setFromScratchTimestamp(elementNum);
+ }
+
@Override
public void evaluate(VectorizedRowBatch batch) {
@@ -63,19 +69,19 @@ public class CastMillisecondsLongToTimestamp extends VectorExpression {
if (inputColVector.isRepeating) {
//All must be selected otherwise size would be zero
//Repeating property will not change.
- outputColVector.setTimestampMilliseconds(0, vector[0]);
+ setMilliseconds(outputColVector, vector, 0);
// Even if there are no nulls, we always copy over entry 0. Simplifies code.
- outputIsNull[0] = inputIsNull[0];
+ outputIsNull[0] = inputIsNull[0];
outputColVector.isRepeating = true;
} else if (inputColVector.noNulls) {
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputColVector.setTimestampMilliseconds(i, vector[i]);
+ setMilliseconds(outputColVector, vector, i);
}
} else {
for(int i = 0; i != n; i++) {
- outputColVector.setTimestampMilliseconds(i, vector[i]);
+ setMilliseconds(outputColVector, vector, i);
}
}
outputColVector.isRepeating = false;
@@ -83,12 +89,12 @@ public class CastMillisecondsLongToTimestamp extends VectorExpression {
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputColVector.setTimestampMilliseconds(i, vector[i]);
+ setMilliseconds(outputColVector, vector, i);
outputIsNull[i] = inputIsNull[i];
}
} else {
for(int i = 0; i != n; i++) {
- outputColVector.setTimestampMilliseconds(i, vector[i]);
+ setMilliseconds(outputColVector, vector, i);
}
System.arraycopy(inputIsNull, 0, outputIsNull, 0, n);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastStringToIntervalDayTime.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastStringToIntervalDayTime.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastStringToIntervalDayTime.java
index a3ddf9f..c8844c8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastStringToIntervalDayTime.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastStringToIntervalDayTime.java
@@ -21,7 +21,7 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions;
import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.IntervalDayTimeColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.serde.serdeConstants;
@@ -56,7 +56,7 @@ public class CastStringToIntervalDayTime extends VectorExpression {
BytesColumnVector inV = (BytesColumnVector) batch.cols[inputColumn];
int[] sel = batch.selected;
int n = batch.size;
- TimestampColumnVector outV = (TimestampColumnVector) batch.cols[outputColumn];
+ IntervalDayTimeColumnVector outV = (IntervalDayTimeColumnVector) batch.cols[outputColumn];
if (n == 0) {
@@ -113,11 +113,11 @@ public class CastStringToIntervalDayTime extends VectorExpression {
}
}
- private void evaluate(TimestampColumnVector outV, BytesColumnVector inV, int i) {
+ private void evaluate(IntervalDayTimeColumnVector outV, BytesColumnVector inV, int i) {
try {
HiveIntervalDayTime interval = HiveIntervalDayTime.valueOf(
new String(inV.vector[i], inV.start[i], inV.length[i], "UTF-8"));
- outV.setEpochSecondsAndSignedNanos(i, interval.getTotalSeconds(), interval.getNanos());
+ outV.set(i, interval);
} catch (Exception e) {
outV.setNullValue(i);
outV.isNull[i] = true;
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToBoolean.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToBoolean.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToBoolean.java
index 55b84b1..b8a58cd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToBoolean.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToBoolean.java
@@ -41,8 +41,8 @@ public class CastTimestampToBoolean extends VectorExpression {
}
private int toBool(TimestampColumnVector timestampColVector, int index) {
- return (timestampColVector.getEpochDay(index) != 0 ||
- timestampColVector.getNanoOfDay(index) != 0) ? 1 : 0;
+ return (timestampColVector.getTime(index) != 0 ||
+ timestampColVector.getNanos(index) != 0) ? 1 : 0;
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToDate.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToDate.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToDate.java
index 00790b9..4e3e62c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToDate.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToDate.java
@@ -44,6 +44,6 @@ public class CastTimestampToDate extends FuncTimestampToLong {
@Override
protected void func(LongColumnVector outV, TimestampColumnVector inV, int i) {
- outV.vector[i] = DateWritable.millisToDays(inV.getTimestampMilliseconds(i));
+ outV.vector[i] = DateWritable.millisToDays(inV.getTime(i));
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToDecimal.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToDecimal.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToDecimal.java
index aec104e..e5bfb15 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToDecimal.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToDecimal.java
@@ -18,9 +18,9 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions;
-import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
+import org.apache.hadoop.hive.serde2.io.TimestampWritable;
/**
* To be used to cast timestamp to decimal.
@@ -39,11 +39,6 @@ public class CastTimestampToDecimal extends FuncTimestampToDecimal {
@Override
protected void func(DecimalColumnVector outV, TimestampColumnVector inV, int i) {
-
- // The BigDecimal class recommends not converting directly from double to BigDecimal,
- // so we convert like the non-vectorized case and got through a string...
- Double timestampDouble = inV.getTimestampSecondsWithFractionalNanos(i);
- HiveDecimal result = HiveDecimal.create(timestampDouble.toString());
- outV.set(i, result);
+ outV.set(i, TimestampWritable.getHiveDecimal(inV.asScratchTimestamp(i)));
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToDouble.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToDouble.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToDouble.java
index f8737f9..a955d79 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToDouble.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToDouble.java
@@ -20,8 +20,7 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.*;
-import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
-import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
+import org.apache.hadoop.hive.serde2.io.TimestampWritable;
public class CastTimestampToDouble extends VectorExpression {
private static final long serialVersionUID = 1L;
@@ -63,7 +62,7 @@ public class CastTimestampToDouble extends VectorExpression {
if (inputColVector.isRepeating) {
//All must be selected otherwise size would be zero
//Repeating property will not change.
- outputVector[0] = inputColVector.getTimestampSecondsWithFractionalNanos(0);
+ outputVector[0] = inputColVector.getDouble(0);
// Even if there are no nulls, we always copy over entry 0. Simplifies code.
outputIsNull[0] = inputIsNull[0];
outputColVector.isRepeating = true;
@@ -71,11 +70,11 @@ public class CastTimestampToDouble extends VectorExpression {
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputVector[i] = inputColVector.getTimestampSecondsWithFractionalNanos(i);
+ outputVector[i] = inputColVector.getDouble(i);
}
} else {
for(int i = 0; i != n; i++) {
- outputVector[i] = inputColVector.getTimestampSecondsWithFractionalNanos(i);
+ outputVector[i] = inputColVector.getDouble(i);
}
}
outputColVector.isRepeating = false;
@@ -83,12 +82,12 @@ public class CastTimestampToDouble extends VectorExpression {
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputVector[i] = inputColVector.getTimestampSecondsWithFractionalNanos(i);
+ outputVector[i] = inputColVector.getDouble(i);
outputIsNull[i] = inputIsNull[i];
}
} else {
for(int i = 0; i != n; i++) {
- outputVector[i] = inputColVector.getTimestampSecondsWithFractionalNanos(i);
+ outputVector[i] = inputColVector.getDouble(i);
}
System.arraycopy(inputIsNull, 0, outputIsNull, 0, n);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToLong.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToLong.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToLong.java
index 4f53f5c..ba2e823 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToLong.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToLong.java
@@ -64,19 +64,19 @@ public class CastTimestampToLong extends VectorExpression {
if (inputColVector.isRepeating) {
//All must be selected otherwise size would be zero
//Repeating property will not change.
- outputVector[0] = inputColVector.getEpochSeconds(0);
+ outputVector[0] = inputColVector.getTimestampAsLong(0);
// Even if there are no nulls, we always copy over entry 0. Simplifies code.
- outputIsNull[0] = inputIsNull[0];
+ outputIsNull[0] = inputIsNull[0];
outputColVector.isRepeating = true;
} else if (inputColVector.noNulls) {
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputVector[i] = inputColVector.getEpochSeconds(i);
+ outputVector[i] = inputColVector.getTimestampAsLong(i);
}
} else {
for(int i = 0; i != n; i++) {
- outputVector[i] = inputColVector.getEpochSeconds(i);
+ outputVector[i] = inputColVector.getTimestampAsLong(i);
}
}
outputColVector.isRepeating = false;
@@ -84,12 +84,12 @@ public class CastTimestampToLong extends VectorExpression {
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputVector[i] = inputColVector.getEpochSeconds(i);
+ outputVector[i] = inputColVector.getTimestampAsLong(i);
outputIsNull[i] = inputIsNull[i];
}
} else {
for(int i = 0; i != n; i++) {
- outputVector[i] = inputColVector.getEpochSeconds(i);
+ outputVector[i] = inputColVector.getTimestampAsLong(i);
}
System.arraycopy(inputIsNull, 0, outputIsNull, 0, n);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ConstantVectorExpression.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ConstantVectorExpression.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ConstantVectorExpression.java
index 24ee9bc..8a743f6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ConstantVectorExpression.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ConstantVectorExpression.java
@@ -24,13 +24,9 @@ import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.common.type.HiveChar;
import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
import org.apache.hadoop.hive.common.type.HiveVarchar;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
import org.apache.hadoop.hive.ql.exec.vector.*;
-import org.apache.hadoop.hive.ql.exec.vector.ColumnVector.Type;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
-import org.apache.hive.common.util.DateUtils;
/**
* Constant is represented as a vector with repeating values.
@@ -44,7 +40,8 @@ public class ConstantVectorExpression extends VectorExpression {
private double doubleValue = 0;
private byte[] bytesValue = null;
private HiveDecimal decimalValue = null;
- private PisaTimestamp timestampValue = null;
+ private Timestamp timestampValue = null;
+ private HiveIntervalDayTime intervalDayTimeValue = null;
private boolean isNullValue = false;
private ColumnVector.Type type;
@@ -97,7 +94,7 @@ public class ConstantVectorExpression extends VectorExpression {
}
public ConstantVectorExpression(int outputColumn, HiveIntervalDayTime value) {
- this(outputColumn, "timestamp");
+ this(outputColumn, "interval_day_time");
setIntervalDayTimeValue(value);
}
@@ -165,6 +162,17 @@ public class ConstantVectorExpression extends VectorExpression {
}
}
+ private void evaluateIntervalDayTime(VectorizedRowBatch vrg) {
+ IntervalDayTimeColumnVector dcv = (IntervalDayTimeColumnVector) vrg.cols[outputColumn];
+ dcv.isRepeating = true;
+ dcv.noNulls = !isNullValue;
+ if (!isNullValue) {
+ dcv.set(0, intervalDayTimeValue);
+ } else {
+ dcv.isNull[0] = true;
+ }
+ }
+
@Override
public void evaluate(VectorizedRowBatch vrg) {
switch (type) {
@@ -183,6 +191,9 @@ public class ConstantVectorExpression extends VectorExpression {
case TIMESTAMP:
evaluateTimestamp(vrg);
break;
+ case INTERVAL_DAY_TIME:
+ evaluateIntervalDayTime(vrg);
+ break;
}
}
@@ -225,16 +236,19 @@ public class ConstantVectorExpression extends VectorExpression {
}
public void setTimestampValue(Timestamp timestampValue) {
- this.timestampValue = new PisaTimestamp(timestampValue);
+ this.timestampValue = timestampValue;
}
- public void setIntervalDayTimeValue(HiveIntervalDayTime intervalDayTimeValue) {
- this.timestampValue = intervalDayTimeValue.pisaTimestampUpdate(new PisaTimestamp());
+ public Timestamp getTimestampValue() {
+ return timestampValue;
}
+ public void setIntervalDayTimeValue(HiveIntervalDayTime intervalDayTimeValue) {
+ this.intervalDayTimeValue = intervalDayTimeValue;
+ }
- public PisaTimestamp getTimestampValue() {
- return timestampValue;
+ public HiveIntervalDayTime getIntervalDayTimeValue() {
+ return intervalDayTimeValue;
}
public String getTypeString() {
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DateColSubtractDateColumn.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DateColSubtractDateColumn.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DateColSubtractDateColumn.java
index 8d2a186..fafacce 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DateColSubtractDateColumn.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DateColSubtractDateColumn.java
@@ -18,7 +18,9 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
+import java.sql.Timestamp;
+
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil;
import org.apache.hadoop.hive.ql.exec.vector.*;
@@ -26,7 +28,7 @@ import org.apache.hadoop.hive.ql.util.DateTimeMath;
import org.apache.hadoop.hive.serde2.io.DateWritable;
// A type date (LongColumnVector storing epoch days) minus a type date produces a
-// type interval_day_time (TimestampColumnVector storing nanosecond interval in 2 longs).
+// type interval_day_time (IntervalDayTimeColumnVector storing nanosecond interval in 2 longs).
public class DateColSubtractDateColumn extends VectorExpression {
private static final long serialVersionUID = 1L;
@@ -34,16 +36,16 @@ public class DateColSubtractDateColumn extends VectorExpression {
private int colNum1;
private int colNum2;
private int outputColumn;
- private PisaTimestamp scratchPisaTimestamp1;
- private PisaTimestamp scratchPisaTimestamp2;
+ private Timestamp scratchTimestamp1;
+ private Timestamp scratchTimestamp2;
private DateTimeMath dtm = new DateTimeMath();
public DateColSubtractDateColumn(int colNum1, int colNum2, int outputColumn) {
this.colNum1 = colNum1;
this.colNum2 = colNum2;
this.outputColumn = outputColumn;
- scratchPisaTimestamp1 = new PisaTimestamp();
- scratchPisaTimestamp2 = new PisaTimestamp();
+ scratchTimestamp1 = new Timestamp(0);
+ scratchTimestamp2 = new Timestamp(0);
}
public DateColSubtractDateColumn() {
@@ -63,7 +65,7 @@ public class DateColSubtractDateColumn extends VectorExpression {
LongColumnVector inputColVector2 = (LongColumnVector) batch.cols[colNum2];
// Output is type interval_day_time.
- TimestampColumnVector outputColVector = (TimestampColumnVector) batch.cols[outputColumn];
+ IntervalDayTimeColumnVector outputColVector = (IntervalDayTimeColumnVector) batch.cols[outputColumn];
int[] sel = batch.selected;
int n = batch.size;
@@ -80,73 +82,69 @@ public class DateColSubtractDateColumn extends VectorExpression {
|| inputColVector1.isRepeating && !inputColVector1.noNulls && inputColVector1.isNull[0]
|| inputColVector2.isRepeating && !inputColVector2.noNulls && inputColVector2.isNull[0];
- // Handle nulls first
+ // Handle nulls first
NullUtil.propagateNullsColCol(
inputColVector1, inputColVector2, outputColVector, sel, n, batch.selectedInUse);
+ HiveIntervalDayTime resultIntervalDayTime = outputColVector.getScratchIntervalDayTime();
+
/* Disregard nulls for processing. In other words,
* the arithmetic operation is performed even if one or
* more inputs are null. This is to improve speed by avoiding
* conditional checks in the inner loop.
*/
if (inputColVector1.isRepeating && inputColVector2.isRepeating) {
- outputColVector.subtract(
- scratchPisaTimestamp1.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector1[0])),
- scratchPisaTimestamp2.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector2[0])),
- 0);
+ scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[0]));
+ scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[0]));
+ dtm.subtract(scratchTimestamp1, scratchTimestamp2, outputColVector.getScratchIntervalDayTime());
+ outputColVector.setFromScratchIntervalDayTime(0);
} else if (inputColVector1.isRepeating) {
+ scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[0]));
if (batch.selectedInUse) {
- scratchPisaTimestamp1.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector1[0]));
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputColVector.subtract(
- scratchPisaTimestamp1,
- scratchPisaTimestamp2.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector2[i])),
- i);
+ scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[i]));
+ dtm.subtract(scratchTimestamp1, scratchTimestamp2, outputColVector.getScratchIntervalDayTime());
+ outputColVector.setFromScratchIntervalDayTime(i);
}
} else {
- scratchPisaTimestamp1.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector1[0]));
for(int i = 0; i != n; i++) {
- outputColVector.subtract(
- scratchPisaTimestamp1,
- scratchPisaTimestamp2.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector2[i])),
- i);
+ scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[i]));
+ dtm.subtract(scratchTimestamp1, scratchTimestamp2, outputColVector.getScratchIntervalDayTime());
+ outputColVector.setFromScratchIntervalDayTime(i);
}
}
} else if (inputColVector2.isRepeating) {
+ scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[0]));
if (batch.selectedInUse) {
- scratchPisaTimestamp2.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector2[0]));
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputColVector.subtract(
- scratchPisaTimestamp1.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector1[i])),
- scratchPisaTimestamp2,
- i);
+ scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[i]));
+ dtm.subtract(scratchTimestamp1, scratchTimestamp2, outputColVector.getScratchIntervalDayTime());
+ outputColVector.setFromScratchIntervalDayTime(i);
}
} else {
- scratchPisaTimestamp2.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector2[0]));
for(int i = 0; i != n; i++) {
- outputColVector.subtract(
- scratchPisaTimestamp1.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector1[i])),
- scratchPisaTimestamp2,
- i);
+ scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[i]));
+ dtm.subtract(scratchTimestamp1, scratchTimestamp2, outputColVector.getScratchIntervalDayTime());
+ outputColVector.setFromScratchIntervalDayTime(i);
}
}
} else {
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputColVector.subtract(
- scratchPisaTimestamp1.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector1[i])),
- scratchPisaTimestamp2.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector2[i])),
- i);
+ scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[i]));
+ scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[i]));
+ dtm.subtract(scratchTimestamp1, scratchTimestamp2, outputColVector.getScratchIntervalDayTime());
+ outputColVector.setFromScratchIntervalDayTime(i);
}
} else {
for(int i = 0; i != n; i++) {
- outputColVector.subtract(
- scratchPisaTimestamp1.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector1[i])),
- scratchPisaTimestamp2.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector2[i])),
- i);
+ scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[i]));
+ scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[i]));
+ dtm.subtract(scratchTimestamp1, scratchTimestamp2, outputColVector.getScratchIntervalDayTime());
+ outputColVector.setFromScratchIntervalDayTime(i);
}
}
}
@@ -157,7 +155,7 @@ public class DateColSubtractDateColumn extends VectorExpression {
* in complex arithmetic expressions like col2 / (col1 - 1)
* in the case when some col1 entries are null.
*/
- NullUtil.setNullDataEntriesTimestamp(outputColVector, batch.selectedInUse, sel, n);
+ NullUtil.setNullDataEntriesIntervalDayTime(outputColVector, batch.selectedInUse, sel, n);
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DateColSubtractDateScalar.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DateColSubtractDateScalar.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DateColSubtractDateScalar.java
index 3ea9331..a9ca93c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DateColSubtractDateScalar.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DateColSubtractDateScalar.java
@@ -18,7 +18,8 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
+import java.sql.Timestamp;
+
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
@@ -35,16 +36,17 @@ public class DateColSubtractDateScalar extends VectorExpression {
private static final long serialVersionUID = 1L;
private int colNum;
- private PisaTimestamp value;
+ private Timestamp value;
private int outputColumn;
- private PisaTimestamp scratchPisaTimestamp;
+ private Timestamp scratchTimestamp1;
private DateTimeMath dtm = new DateTimeMath();
public DateColSubtractDateScalar(int colNum, long value, int outputColumn) {
this.colNum = colNum;
- this.value = new PisaTimestamp().updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) value));
+ this.value = new Timestamp(0);
+ this.value.setTime(DateWritable.daysToMillis((int) value));
this.outputColumn = outputColumn;
- scratchPisaTimestamp = new PisaTimestamp();
+ scratchTimestamp1 = new Timestamp(0);
}
public DateColSubtractDateScalar() {
@@ -60,8 +62,8 @@ public class DateColSubtractDateScalar extends VectorExpression {
// Input #1 is type date (epochDays).
LongColumnVector inputColVector1 = (LongColumnVector) batch.cols[colNum];
- // Output is type Timestamp.
- TimestampColumnVector outputColVector = (TimestampColumnVector) batch.cols[outputColumn];
+ // Output is type HiveIntervalDayTime.
+ IntervalDayTimeColumnVector outputColVector = (IntervalDayTimeColumnVector) batch.cols[outputColumn];
int[] sel = batch.selected;
boolean[] inputIsNull = inputColVector1.isNull;
@@ -77,45 +79,40 @@ public class DateColSubtractDateScalar extends VectorExpression {
}
if (inputColVector1.isRepeating) {
- outputColVector.subtract(
- scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector1[0])),
- value,
- 0);
+ scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[0]));
+ dtm.subtract(scratchTimestamp1, value, outputColVector.getScratchIntervalDayTime());
+ outputColVector.setFromScratchIntervalDayTime(0);
// Even if there are no nulls, we always copy over entry 0. Simplifies code.
outputIsNull[0] = inputIsNull[0];
} else if (inputColVector1.noNulls) {
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputColVector.subtract(
- scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector1[0])),
- value,
- i);
+ scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[i]));
+ dtm.subtract(scratchTimestamp1, value, outputColVector.getScratchIntervalDayTime());
+ outputColVector.setFromScratchIntervalDayTime(i);
}
} else {
for(int i = 0; i != n; i++) {
- outputColVector.subtract(
- scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector1[0])),
- value,
- i);
+ scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[i]));
+ dtm.subtract(scratchTimestamp1, value, outputColVector.getScratchIntervalDayTime());
+ outputColVector.setFromScratchIntervalDayTime(i);
}
}
} else /* there are nulls */ {
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputColVector.subtract(
- scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector1[0])),
- value,
- i);
+ scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[i]));
+ dtm.subtract(scratchTimestamp1, value, outputColVector.getScratchIntervalDayTime());
+ outputColVector.setFromScratchIntervalDayTime(i);
outputIsNull[i] = inputIsNull[i];
}
} else {
for(int i = 0; i != n; i++) {
- outputColVector.subtract(
- scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector1[0])),
- value,
- i);
+ scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[i]));
+ dtm.subtract(scratchTimestamp1, value, outputColVector.getScratchIntervalDayTime());
+ outputColVector.setFromScratchIntervalDayTime(i);
}
System.arraycopy(inputIsNull, 0, outputIsNull, 0, n);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DateScalarSubtractDateColumn.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DateScalarSubtractDateColumn.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DateScalarSubtractDateColumn.java
index a8cabb8..59cf9da 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DateScalarSubtractDateColumn.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DateScalarSubtractDateColumn.java
@@ -18,7 +18,8 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
+import java.sql.Timestamp;
+
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.*;
import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil;
@@ -32,16 +33,17 @@ public class DateScalarSubtractDateColumn extends VectorExpression {
private static final long serialVersionUID = 1L;
private int colNum;
- private PisaTimestamp value;
+ private Timestamp value;
private int outputColumn;
- private PisaTimestamp scratchPisaTimestamp;
+ private Timestamp scratchTimestamp2;
private DateTimeMath dtm = new DateTimeMath();
public DateScalarSubtractDateColumn(long value, int colNum, int outputColumn) {
this.colNum = colNum;
- this.value = new PisaTimestamp().updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) value));
+ this.value = new Timestamp(0);
+ this.value.setTime(DateWritable.daysToMillis((int) value));
this.outputColumn = outputColumn;
- scratchPisaTimestamp = new PisaTimestamp();
+ scratchTimestamp2 = new Timestamp(0);
}
public DateScalarSubtractDateColumn() {
@@ -62,8 +64,8 @@ public class DateScalarSubtractDateColumn extends VectorExpression {
// Input #2 is type date (epochDays).
LongColumnVector inputColVector2 = (LongColumnVector) batch.cols[colNum];
- // Output is type Timestamp.
- TimestampColumnVector outputColVector = (TimestampColumnVector) batch.cols[outputColumn];
+ // Output is type HiveIntervalDayTime.
+ IntervalDayTimeColumnVector outputColVector = (IntervalDayTimeColumnVector) batch.cols[outputColumn];
int[] sel = batch.selected;
boolean[] inputIsNull = inputColVector2.isNull;
@@ -80,46 +82,40 @@ public class DateScalarSubtractDateColumn extends VectorExpression {
}
if (inputColVector2.isRepeating) {
- outputColVector.subtract(
- value,
- scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector2[0])),
- 0);
-
+ scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[0]));
+ dtm.subtract(value, scratchTimestamp2, outputColVector.getScratchIntervalDayTime());
+ outputColVector.setFromScratchIntervalDayTime(0);
// Even if there are no nulls, we always copy over entry 0. Simplifies code.
outputIsNull[0] = inputIsNull[0];
} else if (inputColVector2.noNulls) {
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputColVector.subtract(
- value,
- scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector2[i])),
- i);
+ scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[i]));
+ dtm.subtract(value, scratchTimestamp2, outputColVector.getScratchIntervalDayTime());
+ outputColVector.setFromScratchIntervalDayTime(i);
}
} else {
for(int i = 0; i != n; i++) {
- outputColVector.subtract(
- value,
- scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector2[i])),
- i);
+ scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[i]));
+ dtm.subtract(value, scratchTimestamp2, outputColVector.getScratchIntervalDayTime());
+ outputColVector.setFromScratchIntervalDayTime(i);
}
}
} else { /* there are nulls */
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputColVector.subtract(
- value,
- scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector2[i])),
- i);
+ scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[i]));
+ dtm.subtract(value, scratchTimestamp2, outputColVector.getScratchIntervalDayTime());
+ outputColVector.setFromScratchIntervalDayTime(i);
outputIsNull[i] = inputIsNull[i];
}
} else {
for(int i = 0; i != n; i++) {
- outputColVector.subtract(
- value,
- scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector2[i])),
- i);
+ scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[i]));
+ dtm.subtract(value, scratchTimestamp2, outputColVector.getScratchIntervalDayTime());
+ outputColVector.setFromScratchIntervalDayTime(i);
}
System.arraycopy(inputIsNull, 0, outputIsNull, 0, n);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterTimestampColumnInList.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterTimestampColumnInList.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterTimestampColumnInList.java
index 42e4984..25a276a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterTimestampColumnInList.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterTimestampColumnInList.java
@@ -21,7 +21,6 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions;
import java.sql.Timestamp;
import java.util.HashSet;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor.Descriptor;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
@@ -35,7 +34,7 @@ public class FilterTimestampColumnInList extends VectorExpression implements ITi
private Timestamp[] inListValues;
// The set object containing the IN list.
- private transient HashSet<PisaTimestamp> inSet;
+ private transient HashSet<Timestamp> inSet;
public FilterTimestampColumnInList() {
super();
@@ -58,9 +57,9 @@ public class FilterTimestampColumnInList extends VectorExpression implements ITi
}
if (inSet == null) {
- inSet = new HashSet<PisaTimestamp>(inListValues.length);
+ inSet = new HashSet<Timestamp>(inListValues.length);
for (Timestamp val : inListValues) {
- inSet.add(new PisaTimestamp(val));
+ inSet.add(val);
}
}
@@ -74,16 +73,13 @@ public class FilterTimestampColumnInList extends VectorExpression implements ITi
return;
}
- PisaTimestamp scratchTimestamp = new PisaTimestamp();
-
if (inputColVector.noNulls) {
if (inputColVector.isRepeating) {
// All must be selected otherwise size would be zero
// Repeating property will not change.
- inputColVector.pisaTimestampUpdate(scratchTimestamp, 0);
- if (!(inSet.contains(scratchTimestamp))) {
+ if (!(inSet.contains(inputColVector.asScratchTimestamp(0)))) {
//Entire batch is filtered out.
batch.size = 0;
}
@@ -91,8 +87,7 @@ public class FilterTimestampColumnInList extends VectorExpression implements ITi
int newSize = 0;
for(int j = 0; j != n; j++) {
int i = sel[j];
- inputColVector.pisaTimestampUpdate(scratchTimestamp, i);
- if (inSet.contains(scratchTimestamp)) {
+ if (inSet.contains(inputColVector.asScratchTimestamp(i))) {
sel[newSize++] = i;
}
}
@@ -100,8 +95,7 @@ public class FilterTimestampColumnInList extends VectorExpression implements ITi
} else {
int newSize = 0;
for(int i = 0; i != n; i++) {
- inputColVector.pisaTimestampUpdate(scratchTimestamp, i);
- if (inSet.contains(scratchTimestamp)) {
+ if (inSet.contains(inputColVector.asScratchTimestamp(i))) {
sel[newSize++] = i;
}
}
@@ -116,8 +110,7 @@ public class FilterTimestampColumnInList extends VectorExpression implements ITi
//All must be selected otherwise size would be zero
//Repeating property will not change.
if (!nullPos[0]) {
- inputColVector.pisaTimestampUpdate(scratchTimestamp, 0);
- if (!inSet.contains(scratchTimestamp)) {
+ if (!inSet.contains(inputColVector.asScratchTimestamp(0))) {
//Entire batch is filtered out.
batch.size = 0;
@@ -130,8 +123,7 @@ public class FilterTimestampColumnInList extends VectorExpression implements ITi
for(int j = 0; j != n; j++) {
int i = sel[j];
if (!nullPos[i]) {
- inputColVector.pisaTimestampUpdate(scratchTimestamp, i);
- if (inSet.contains(scratchTimestamp)) {
+ if (inSet.contains(inputColVector.asScratchTimestamp(i))) {
sel[newSize++] = i;
}
}
@@ -143,8 +135,7 @@ public class FilterTimestampColumnInList extends VectorExpression implements ITi
int newSize = 0;
for(int i = 0; i != n; i++) {
if (!nullPos[i]) {
- inputColVector.pisaTimestampUpdate(scratchTimestamp, i);
- if (inSet.contains(scratchTimestamp)) {
+ if (inSet.contains(inputColVector.asScratchTimestamp(i))) {
sel[newSize++] = i;
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprIntervalDayTimeColumnColumn.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprIntervalDayTimeColumnColumn.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprIntervalDayTimeColumnColumn.java
index a6f8057..804923e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprIntervalDayTimeColumnColumn.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprIntervalDayTimeColumnColumn.java
@@ -17,24 +17,123 @@
*/
package org.apache.hadoop.hive.ql.exec.vector.expressions;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.IntervalDayTimeColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
/**
* Compute IF(expr1, expr2, expr3) for 3 input column expressions.
* The first is always a boolean (LongColumnVector).
* The second and third are long columns or long expression results.
*/
-public class IfExprIntervalDayTimeColumnColumn extends IfExprTimestampColumnColumnBase {
+public class IfExprIntervalDayTimeColumnColumn extends VectorExpression {
private static final long serialVersionUID = 1L;
+ private int arg1Column, arg2Column, arg3Column;
+ private int outputColumn;
+
public IfExprIntervalDayTimeColumnColumn(int arg1Column, int arg2Column, int arg3Column, int outputColumn) {
- super(arg1Column, arg2Column, arg3Column, outputColumn);
+ this.arg1Column = arg1Column;
+ this.arg2Column = arg2Column;
+ this.arg3Column = arg3Column;
+ this.outputColumn = outputColumn;
}
public IfExprIntervalDayTimeColumnColumn() {
super();
}
+ @Override
+ public void evaluate(VectorizedRowBatch batch) {
+
+ if (childExpressions != null) {
+ super.evaluateChildren(batch);
+ }
+
+ LongColumnVector arg1ColVector = (LongColumnVector) batch.cols[arg1Column];
+ IntervalDayTimeColumnVector arg2ColVector = (IntervalDayTimeColumnVector) batch.cols[arg2Column];
+ IntervalDayTimeColumnVector arg3ColVector = (IntervalDayTimeColumnVector) batch.cols[arg3Column];
+ IntervalDayTimeColumnVector outputColVector = (IntervalDayTimeColumnVector) batch.cols[outputColumn];
+ int[] sel = batch.selected;
+ boolean[] outputIsNull = outputColVector.isNull;
+ outputColVector.noNulls = arg2ColVector.noNulls && arg3ColVector.noNulls;
+ outputColVector.isRepeating = false; // may override later
+ int n = batch.size;
+ long[] vector1 = arg1ColVector.vector;
+
+ // return immediately if batch is empty
+ if (n == 0) {
+ return;
+ }
+
+ /* All the code paths below propagate nulls even if neither arg2 nor arg3
+ * have nulls. This is to reduce the number of code paths and shorten the
+ * code, at the expense of maybe doing unnecessary work if neither input
+ * has nulls. This could be improved in the future by expanding the number
+ * of code paths.
+ */
+ if (arg1ColVector.isRepeating) {
+ if (vector1[0] == 1) {
+ arg2ColVector.copySelected(batch.selectedInUse, sel, n, outputColVector);
+ } else {
+ arg3ColVector.copySelected(batch.selectedInUse, sel, n, outputColVector);
+ }
+ return;
+ }
+
+ // extend any repeating values and noNulls indicator in the inputs
+ arg2ColVector.flatten(batch.selectedInUse, sel, n);
+ arg3ColVector.flatten(batch.selectedInUse, sel, n);
+
+ if (arg1ColVector.noNulls) {
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ outputColVector.set(i, vector1[i] == 1 ? arg2ColVector.asScratchIntervalDayTime(i) : arg3ColVector.asScratchIntervalDayTime(i));
+ outputIsNull[i] = (vector1[i] == 1 ?
+ arg2ColVector.isNull[i] : arg3ColVector.isNull[i]);
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ outputColVector.set(i, vector1[i] == 1 ? arg2ColVector.asScratchIntervalDayTime(i) : arg3ColVector.asScratchIntervalDayTime(i));
+ outputIsNull[i] = (vector1[i] == 1 ?
+ arg2ColVector.isNull[i] : arg3ColVector.isNull[i]);
+ }
+ }
+ } else /* there are nulls */ {
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ outputColVector.set(i, !arg1ColVector.isNull[i] && vector1[i] == 1 ?
+ arg2ColVector.asScratchIntervalDayTime(i) : arg3ColVector.asScratchIntervalDayTime(i));
+ outputIsNull[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ?
+ arg2ColVector.isNull[i] : arg3ColVector.isNull[i]);
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ outputColVector.set(i, !arg1ColVector.isNull[i] && vector1[i] == 1 ?
+ arg2ColVector.asScratchIntervalDayTime(i) : arg3ColVector.asScratchIntervalDayTime(i));
+ outputIsNull[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ?
+ arg2ColVector.isNull[i] : arg3ColVector.isNull[i]);
+ }
+ }
+ }
+
+ // restore repeating and no nulls indicators
+ arg2ColVector.unFlatten();
+ arg3ColVector.unFlatten();
+ }
+
+ @Override
+ public int getOutputColumn() {
+ return outputColumn;
+ }
+
+ @Override
+ public String getOutputType() {
+ return "interval_day_time";
+ }
@Override
public VectorExpressionDescriptor.Descriptor getDescriptor() {
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprIntervalDayTimeColumnScalar.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprIntervalDayTimeColumnScalar.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprIntervalDayTimeColumnScalar.java
index 4beb50a..8face7d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprIntervalDayTimeColumnScalar.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprIntervalDayTimeColumnScalar.java
@@ -19,8 +19,10 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions;
import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.IntervalDayTimeColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
/**
* Compute IF(expr1, expr2, expr3) for 3 input column expressions.
@@ -28,13 +30,20 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
* The second is a column or non-constant expression result.
* The third is a constant value.
*/
-public class IfExprIntervalDayTimeColumnScalar extends IfExprTimestampColumnScalarBase {
+public class IfExprIntervalDayTimeColumnScalar extends VectorExpression {
private static final long serialVersionUID = 1L;
+ private int arg1Column, arg2Column;
+ private HiveIntervalDayTime arg3Scalar;
+ private int outputColumn;
+
public IfExprIntervalDayTimeColumnScalar(int arg1Column, int arg2Column, HiveIntervalDayTime arg3Scalar,
int outputColumn) {
- super(arg1Column, arg2Column, arg3Scalar.pisaTimestampUpdate(new PisaTimestamp()), outputColumn);
+ this.arg1Column = arg1Column;
+ this.arg2Column = arg2Column;
+ this.arg3Scalar = arg3Scalar;
+ this.outputColumn = outputColumn;
}
public IfExprIntervalDayTimeColumnScalar() {
@@ -42,6 +51,85 @@ public class IfExprIntervalDayTimeColumnScalar extends IfExprTimestampColumnScal
}
@Override
+ public void evaluate(VectorizedRowBatch batch) {
+
+ if (childExpressions != null) {
+ super.evaluateChildren(batch);
+ }
+
+ LongColumnVector arg1ColVector = (LongColumnVector) batch.cols[arg1Column];
+ IntervalDayTimeColumnVector arg2ColVector = (IntervalDayTimeColumnVector) batch.cols[arg2Column];
+ IntervalDayTimeColumnVector outputColVector = (IntervalDayTimeColumnVector) batch.cols[outputColumn];
+ int[] sel = batch.selected;
+ boolean[] outputIsNull = outputColVector.isNull;
+ outputColVector.noNulls = arg2ColVector.noNulls; // nulls can only come from arg2
+ outputColVector.isRepeating = false; // may override later
+ int n = batch.size;
+ long[] vector1 = arg1ColVector.vector;
+
+ // return immediately if batch is empty
+ if (n == 0) {
+ return;
+ }
+
+ if (arg1ColVector.isRepeating) {
+ if (vector1[0] == 1) {
+ arg2ColVector.copySelected(batch.selectedInUse, sel, n, outputColVector);
+ } else {
+ outputColVector.fill(arg3Scalar);
+ }
+ return;
+ }
+
+ // Extend any repeating values and noNulls indicator in the inputs to
+ // reduce the number of code paths needed below.
+ arg2ColVector.flatten(batch.selectedInUse, sel, n);
+
+ if (arg1ColVector.noNulls) {
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ outputColVector.set(i, vector1[i] == 1 ? arg2ColVector.asScratchIntervalDayTime(i) : arg3Scalar);
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ outputColVector.set(i, vector1[i] == 1 ? arg2ColVector.asScratchIntervalDayTime(i) : arg3Scalar);
+ }
+ }
+ } else /* there are nulls */ {
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ outputColVector.set(i, !arg1ColVector.isNull[i] && vector1[i] == 1 ?
+ arg2ColVector.asScratchIntervalDayTime(i) : arg3Scalar);
+ outputIsNull[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ?
+ arg2ColVector.isNull[i] : false);
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ outputColVector.set(i, !arg1ColVector.isNull[i] && vector1[i] == 1 ?
+ arg2ColVector.asScratchIntervalDayTime(i) : arg3Scalar);
+ outputIsNull[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ?
+ arg2ColVector.isNull[i] : false);
+ }
+ }
+ }
+
+ // restore repeating and no nulls indicators
+ arg2ColVector.unFlatten();
+ }
+
+ @Override
+ public int getOutputColumn() {
+ return outputColumn;
+ }
+
+ @Override
+ public String getOutputType() {
+ return "interval_day_time";
+ }
+
+ @Override
public VectorExpressionDescriptor.Descriptor getDescriptor() {
return (new VectorExpressionDescriptor.Builder())
.setMode(
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprIntervalDayTimeScalarColumn.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprIntervalDayTimeScalarColumn.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprIntervalDayTimeScalarColumn.java
index 5463c7c..40f2e08 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprIntervalDayTimeScalarColumn.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprIntervalDayTimeScalarColumn.java
@@ -19,8 +19,10 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions;
import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.IntervalDayTimeColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
/**
* Compute IF(expr1, expr2, expr3) for 3 input column expressions.
@@ -28,13 +30,20 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
* The second is a column or non-constant expression result.
* The third is a constant value.
*/
-public class IfExprIntervalDayTimeScalarColumn extends IfExprTimestampScalarColumnBase {
+public class IfExprIntervalDayTimeScalarColumn extends VectorExpression {
private static final long serialVersionUID = 1L;
+ private int arg1Column, arg3Column;
+ private HiveIntervalDayTime arg2Scalar;
+ private int outputColumn;
+
public IfExprIntervalDayTimeScalarColumn(int arg1Column, HiveIntervalDayTime arg2Scalar, int arg3Column,
int outputColumn) {
- super(arg1Column, arg2Scalar.pisaTimestampUpdate(new PisaTimestamp()), arg3Column, outputColumn);
+ this.arg1Column = arg1Column;
+ this.arg2Scalar = arg2Scalar;
+ this.arg3Column = arg3Column;
+ this.outputColumn = outputColumn;
}
public IfExprIntervalDayTimeScalarColumn() {
@@ -42,6 +51,87 @@ public class IfExprIntervalDayTimeScalarColumn extends IfExprTimestampScalarColu
}
@Override
+ public void evaluate(VectorizedRowBatch batch) {
+
+ if (childExpressions != null) {
+ super.evaluateChildren(batch);
+ }
+
+ LongColumnVector arg1ColVector = (LongColumnVector) batch.cols[arg1Column];
+ IntervalDayTimeColumnVector arg3ColVector = (IntervalDayTimeColumnVector) batch.cols[arg3Column];
+ IntervalDayTimeColumnVector outputColVector = (IntervalDayTimeColumnVector) batch.cols[outputColumn];
+ int[] sel = batch.selected;
+ boolean[] outputIsNull = outputColVector.isNull;
+ outputColVector.noNulls = arg3ColVector.noNulls; // nulls can only come from arg3 column vector
+ outputColVector.isRepeating = false; // may override later
+ int n = batch.size;
+ long[] vector1 = arg1ColVector.vector;
+
+ // return immediately if batch is empty
+ if (n == 0) {
+ return;
+ }
+
+ if (arg1ColVector.isRepeating) {
+ if (vector1[0] == 1) {
+ outputColVector.fill(arg2Scalar);
+ } else {
+ arg3ColVector.copySelected(batch.selectedInUse, sel, n, outputColVector);
+ }
+ return;
+ }
+
+ // Extend any repeating values and noNulls indicator in the inputs to
+ // reduce the number of code paths needed below.
+ // This could be optimized in the future by having separate paths
+ // for when arg3ColVector is repeating or has no nulls.
+ arg3ColVector.flatten(batch.selectedInUse, sel, n);
+
+ if (arg1ColVector.noNulls) {
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ outputColVector.set(i, vector1[i] == 1 ? arg2Scalar : arg3ColVector.asScratchIntervalDayTime(i));
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ outputColVector.set(i, vector1[i] == 1 ? arg2Scalar : arg3ColVector.asScratchIntervalDayTime(i));
+ }
+ }
+ } else /* there are nulls */ {
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ outputColVector.set(i, !arg1ColVector.isNull[i] && vector1[i] == 1 ?
+ arg2Scalar : arg3ColVector.asScratchIntervalDayTime(i));
+ outputIsNull[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ?
+ false : arg3ColVector.isNull[i]);
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ outputColVector.set(i, !arg1ColVector.isNull[i] && vector1[i] == 1 ?
+ arg2Scalar : arg3ColVector.asScratchIntervalDayTime(i));
+ outputIsNull[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ?
+ false : arg3ColVector.isNull[i]);
+ }
+ }
+ }
+
+ // restore repeating and no nulls indicators
+ arg3ColVector.unFlatten();
+ }
+
+ @Override
+ public int getOutputColumn() {
+ return outputColumn;
+ }
+
+ @Override
+ public String getOutputType() {
+ return "interval_day_time";
+ }
+
+ @Override
public VectorExpressionDescriptor.Descriptor getDescriptor() {
return (new VectorExpressionDescriptor.Builder())
.setMode(
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprIntervalDayTimeScalarScalar.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprIntervalDayTimeScalarScalar.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprIntervalDayTimeScalarScalar.java
index af2e0c0..43676dd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprIntervalDayTimeScalarScalar.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprIntervalDayTimeScalarScalar.java
@@ -18,9 +18,13 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions;
+import java.util.Arrays;
+
import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.IntervalDayTimeColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
/**
* Compute IF(expr1, expr2, expr3) for 3 input expressions.
@@ -28,13 +32,21 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
* The second is a constant value.
* The third is a constant value.
*/
-public class IfExprIntervalDayTimeScalarScalar extends IfExprTimestampScalarScalarBase {
+public class IfExprIntervalDayTimeScalarScalar extends VectorExpression {
private static final long serialVersionUID = 1L;
+ private int arg1Column;
+ private HiveIntervalDayTime arg2Scalar;
+ private HiveIntervalDayTime arg3Scalar;
+ private int outputColumn;
+
public IfExprIntervalDayTimeScalarScalar(int arg1Column, HiveIntervalDayTime arg2Scalar, HiveIntervalDayTime arg3Scalar,
int outputColumn) {
- super(arg1Column, arg2Scalar.pisaTimestampUpdate(new PisaTimestamp()), arg3Scalar.pisaTimestampUpdate(new PisaTimestamp()), outputColumn);
+ this.arg1Column = arg1Column;
+ this.arg2Scalar = arg2Scalar;
+ this.arg3Scalar = arg3Scalar;
+ this.outputColumn = outputColumn;
}
public IfExprIntervalDayTimeScalarScalar() {
@@ -42,6 +54,72 @@ public class IfExprIntervalDayTimeScalarScalar extends IfExprTimestampScalarScal
}
@Override
+ public void evaluate(VectorizedRowBatch batch) {
+
+ if (childExpressions != null) {
+ super.evaluateChildren(batch);
+ }
+
+ LongColumnVector arg1ColVector = (LongColumnVector) batch.cols[arg1Column];
+ IntervalDayTimeColumnVector outputColVector = (IntervalDayTimeColumnVector) batch.cols[outputColumn];
+ int[] sel = batch.selected;
+ boolean[] outputIsNull = outputColVector.isNull;
+ outputColVector.noNulls = false; // output is a scalar which we know is non null
+ outputColVector.isRepeating = false; // may override later
+ int n = batch.size;
+ long[] vector1 = arg1ColVector.vector;
+
+ // return immediately if batch is empty
+ if (n == 0) {
+ return;
+ }
+
+ if (arg1ColVector.isRepeating) {
+ if (vector1[0] == 1) {
+ outputColVector.fill(arg2Scalar);
+ } else {
+ outputColVector.fill(arg3Scalar);
+ }
+ } else if (arg1ColVector.noNulls) {
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ outputColVector.set(i, vector1[i] == 1 ? arg2Scalar : arg3Scalar);
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ outputColVector.set(i, vector1[i] == 1 ? arg2Scalar : arg3Scalar);
+ }
+ }
+ } else /* there are nulls */ {
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ outputColVector.set(i, !arg1ColVector.isNull[i] && vector1[i] == 1 ?
+ arg2Scalar : arg3Scalar);
+ outputIsNull[i] = false;
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ outputColVector.set(i, !arg1ColVector.isNull[i] && vector1[i] == 1 ?
+ arg2Scalar : arg3Scalar);
+ }
+ Arrays.fill(outputIsNull, 0, n, false);
+ }
+ }
+ }
+
+ @Override
+ public int getOutputColumn() {
+ return outputColumn;
+ }
+
+ @Override
+ public String getOutputType() {
+ return "timestamp";
+ }
+
+ @Override
public VectorExpressionDescriptor.Descriptor getDescriptor() {
return (new VectorExpressionDescriptor.Builder())
.setMode(
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampColumnColumnBase.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampColumnColumnBase.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampColumnColumnBase.java
index d3dd67d..8441863 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampColumnColumnBase.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampColumnColumnBase.java
@@ -89,13 +89,13 @@ public abstract class IfExprTimestampColumnColumnBase extends VectorExpression {
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputColVector.set(i, vector1[i] == 1 ? arg2ColVector.asScratchPisaTimestamp(i) : arg3ColVector.asScratchPisaTimestamp(i));
+ outputColVector.set(i, vector1[i] == 1 ? arg2ColVector.asScratchTimestamp(i) : arg3ColVector.asScratchTimestamp(i));
outputIsNull[i] = (vector1[i] == 1 ?
arg2ColVector.isNull[i] : arg3ColVector.isNull[i]);
}
} else {
for(int i = 0; i != n; i++) {
- outputColVector.set(i, vector1[i] == 1 ? arg2ColVector.asScratchPisaTimestamp(i) : arg3ColVector.asScratchPisaTimestamp(i));
+ outputColVector.set(i, vector1[i] == 1 ? arg2ColVector.asScratchTimestamp(i) : arg3ColVector.asScratchTimestamp(i));
outputIsNull[i] = (vector1[i] == 1 ?
arg2ColVector.isNull[i] : arg3ColVector.isNull[i]);
}
@@ -105,14 +105,14 @@ public abstract class IfExprTimestampColumnColumnBase extends VectorExpression {
for(int j = 0; j != n; j++) {
int i = sel[j];
outputColVector.set(i, !arg1ColVector.isNull[i] && vector1[i] == 1 ?
- arg2ColVector.asScratchPisaTimestamp(i) : arg3ColVector.asScratchPisaTimestamp(i));
+ arg2ColVector.asScratchTimestamp(i) : arg3ColVector.asScratchTimestamp(i));
outputIsNull[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ?
arg2ColVector.isNull[i] : arg3ColVector.isNull[i]);
}
} else {
for(int i = 0; i != n; i++) {
outputColVector.set(i, !arg1ColVector.isNull[i] && vector1[i] == 1 ?
- arg2ColVector.asScratchPisaTimestamp(i) : arg3ColVector.asScratchPisaTimestamp(i));
+ arg2ColVector.asScratchTimestamp(i) : arg3ColVector.asScratchTimestamp(i));
outputIsNull[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ?
arg2ColVector.isNull[i] : arg3ColVector.isNull[i]);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampColumnScalar.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampColumnScalar.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampColumnScalar.java
index 0660038..ae997e0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampColumnScalar.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampColumnScalar.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions;
import java.sql.Timestamp;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
/**
@@ -35,7 +34,7 @@ public class IfExprTimestampColumnScalar extends IfExprTimestampColumnScalarBase
public IfExprTimestampColumnScalar(int arg1Column, int arg2Column, Timestamp arg3Scalar,
int outputColumn) {
- super(arg1Column, arg2Column, new PisaTimestamp(arg3Scalar), outputColumn);
+ super(arg1Column, arg2Column, arg3Scalar, outputColumn);
}
public IfExprTimestampColumnScalar() {
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampColumnScalarBase.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampColumnScalarBase.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampColumnScalarBase.java
index 8aaad3f..6b87ff2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampColumnScalarBase.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampColumnScalarBase.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
+import java.sql.Timestamp;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
@@ -38,10 +38,10 @@ public abstract class IfExprTimestampColumnScalarBase extends VectorExpression {
private static final long serialVersionUID = 1L;
private int arg1Column, arg2Column;
- private PisaTimestamp arg3Scalar;
+ private Timestamp arg3Scalar;
private int outputColumn;
- public IfExprTimestampColumnScalarBase(int arg1Column, int arg2Column, PisaTimestamp arg3Scalar,
+ public IfExprTimestampColumnScalarBase(int arg1Column, int arg2Column, Timestamp arg3Scalar,
int outputColumn) {
this.arg1Column = arg1Column;
this.arg2Column = arg2Column;
@@ -91,11 +91,11 @@ public abstract class IfExprTimestampColumnScalarBase extends VectorExpression {
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputColVector.set(i, vector1[i] == 1 ? arg2ColVector.asScratchPisaTimestamp(i) : arg3Scalar);
+ outputColVector.set(i, vector1[i] == 1 ? arg2ColVector.asScratchTimestamp(i) : arg3Scalar);
}
} else {
for(int i = 0; i != n; i++) {
- outputColVector.set(i, vector1[i] == 1 ? arg2ColVector.asScratchPisaTimestamp(i) : arg3Scalar);
+ outputColVector.set(i, vector1[i] == 1 ? arg2ColVector.asScratchTimestamp(i) : arg3Scalar);
}
}
} else /* there are nulls */ {
@@ -103,14 +103,14 @@ public abstract class IfExprTimestampColumnScalarBase extends VectorExpression {
for(int j = 0; j != n; j++) {
int i = sel[j];
outputColVector.set(i, !arg1ColVector.isNull[i] && vector1[i] == 1 ?
- arg2ColVector.asScratchPisaTimestamp(i) : arg3Scalar);
+ arg2ColVector.asScratchTimestamp(i) : arg3Scalar);
outputIsNull[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ?
arg2ColVector.isNull[i] : false);
}
} else {
for(int i = 0; i != n; i++) {
outputColVector.set(i, !arg1ColVector.isNull[i] && vector1[i] == 1 ?
- arg2ColVector.asScratchPisaTimestamp(i) : arg3Scalar);
+ arg2ColVector.asScratchTimestamp(i) : arg3Scalar);
outputIsNull[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ?
arg2ColVector.isNull[i] : false);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampScalarColumn.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampScalarColumn.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampScalarColumn.java
index 7f618cb..3d53df1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampScalarColumn.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampScalarColumn.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions;
import java.sql.Timestamp;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
/**
@@ -35,7 +34,7 @@ public class IfExprTimestampScalarColumn extends IfExprTimestampScalarColumnBase
public IfExprTimestampScalarColumn(int arg1Column, Timestamp arg2Scalar, int arg3Column,
int outputColumn) {
- super(arg1Column, new PisaTimestamp(arg2Scalar), arg3Column, outputColumn);
+ super(arg1Column, arg2Scalar, arg3Column, outputColumn);
}
public IfExprTimestampScalarColumn() {
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampScalarColumnBase.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampScalarColumnBase.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampScalarColumnBase.java
index 84d7655..2162f17 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampScalarColumnBase.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampScalarColumnBase.java
@@ -18,7 +18,8 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
+import java.sql.Timestamp;
+
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
@@ -36,10 +37,10 @@ public abstract class IfExprTimestampScalarColumnBase extends VectorExpression {
private static final long serialVersionUID = 1L;
private int arg1Column, arg3Column;
- private PisaTimestamp arg2Scalar;
+ private Timestamp arg2Scalar;
private int outputColumn;
- public IfExprTimestampScalarColumnBase(int arg1Column, PisaTimestamp arg2Scalar, int arg3Column,
+ public IfExprTimestampScalarColumnBase(int arg1Column, Timestamp arg2Scalar, int arg3Column,
int outputColumn) {
this.arg1Column = arg1Column;
this.arg2Scalar = arg2Scalar;
@@ -91,11 +92,11 @@ public abstract class IfExprTimestampScalarColumnBase extends VectorExpression {
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputColVector.set(i, vector1[i] == 1 ? arg2Scalar : arg3ColVector.asScratchPisaTimestamp(i));
+ outputColVector.set(i, vector1[i] == 1 ? arg2Scalar : arg3ColVector.asScratchTimestamp(i));
}
} else {
for(int i = 0; i != n; i++) {
- outputColVector.set(i, vector1[i] == 1 ? arg2Scalar : arg3ColVector.asScratchPisaTimestamp(i));
+ outputColVector.set(i, vector1[i] == 1 ? arg2Scalar : arg3ColVector.asScratchTimestamp(i));
}
}
} else /* there are nulls */ {
@@ -103,14 +104,14 @@ public abstract class IfExprTimestampScalarColumnBase extends VectorExpression {
for(int j = 0; j != n; j++) {
int i = sel[j];
outputColVector.set(i, !arg1ColVector.isNull[i] && vector1[i] == 1 ?
- arg2Scalar : arg3ColVector.asScratchPisaTimestamp(i));
+ arg2Scalar : arg3ColVector.asScratchTimestamp(i));
outputIsNull[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ?
false : arg3ColVector.isNull[i]);
}
} else {
for(int i = 0; i != n; i++) {
outputColVector.set(i, !arg1ColVector.isNull[i] && vector1[i] == 1 ?
- arg2Scalar : arg3ColVector.asScratchPisaTimestamp(i));
+ arg2Scalar : arg3ColVector.asScratchTimestamp(i));
outputIsNull[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ?
false : arg3ColVector.isNull[i]);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampScalarScalar.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampScalarScalar.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampScalarScalar.java
index 5286ea3..cd00d3a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampScalarScalar.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampScalarScalar.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
import java.sql.Timestamp;
@@ -35,7 +34,7 @@ public class IfExprTimestampScalarScalar extends IfExprTimestampScalarScalarBase
public IfExprTimestampScalarScalar(int arg1Column, Timestamp arg2Scalar, Timestamp arg3Scalar,
int outputColumn) {
- super(arg1Column, new PisaTimestamp(arg2Scalar), new PisaTimestamp(arg3Scalar), outputColumn);
+ super(arg1Column, arg2Scalar, arg3Scalar, outputColumn);
}
public IfExprTimestampScalarScalar() {
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampScalarScalarBase.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampScalarScalarBase.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampScalarScalarBase.java
index 1aeabfc..707f574 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampScalarScalarBase.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampScalarScalarBase.java
@@ -18,11 +18,12 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions;
-import org.apache.hadoop.hive.common.type.PisaTimestamp;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+
+import java.sql.Timestamp;
import java.util.Arrays;
/**
@@ -36,11 +37,11 @@ public abstract class IfExprTimestampScalarScalarBase extends VectorExpression {
private static final long serialVersionUID = 1L;
private int arg1Column;
- private PisaTimestamp arg2Scalar;
- private PisaTimestamp arg3Scalar;
+ private Timestamp arg2Scalar;
+ private Timestamp arg3Scalar;
private int outputColumn;
- public IfExprTimestampScalarScalarBase(int arg1Column, PisaTimestamp arg2Scalar, PisaTimestamp arg3Scalar,
+ public IfExprTimestampScalarScalarBase(int arg1Column, Timestamp arg2Scalar, Timestamp arg3Scalar,
int outputColumn) {
this.arg1Column = arg1Column;
this.arg2Scalar = arg2Scalar;
@@ -116,8 +117,4 @@ public abstract class IfExprTimestampScalarScalarBase extends VectorExpression {
public String getOutputType() {
return "timestamp";
}
-
- public int getArg1Column() {
- return arg1Column;
- }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52016296/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/NullUtil.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/NullUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/NullUtil.java
index 3c6824d..eb493bf 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/NullUtil.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/NullUtil.java
@@ -22,6 +22,7 @@ import java.util.Arrays;
import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.IntervalDayTimeColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
@@ -107,6 +108,31 @@ public class NullUtil {
}
}
+ /**
+ * Set the data value for all NULL entries to the designated NULL_VALUE.
+ */
+ public static void setNullDataEntriesIntervalDayTime(
+ IntervalDayTimeColumnVector v, boolean selectedInUse, int[] sel, int n) {
+ if (v.noNulls) {
+ return;
+ } else if (v.isRepeating && v.isNull[0]) {
+ v.setNullValue(0);
+ } else if (selectedInUse) {
+ for (int j = 0; j != n; j++) {
+ int i = sel[j];
+ if(v.isNull[i]) {
+ v.setNullValue(i);
+ }
+ }
+ } else {
+ for (int i = 0; i != n; i++) {
+ if(v.isNull[i]) {
+ v.setNullValue(i);
+ }
+ }
+ }
+ }
+
// for use by Column-Scalar and Scalar-Column arithmetic for null propagation
public static void setNullOutputEntriesColScalar(
ColumnVector v, boolean selectedInUse, int[] sel, int n) {