You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by om...@apache.org on 2015/11/20 22:46:41 UTC
[01/12] hive git commit: HIVE-12300 : deprecate MR in Hive 2.0
(Sergey Shelukhin, reviewed by Sushanth Sowmyan)
Repository: hive
Updated Branches:
refs/heads/master-fixed 6a3fa6cc3 -> 4a13e5fb2
HIVE-12300 : deprecate MR in Hive 2.0 (Sergey Shelukhin, reviewed by Sushanth Sowmyan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/82705f04
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/82705f04
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/82705f04
Branch: refs/heads/master-fixed
Commit: 82705f04d12122d974ea9ab20210beeb359480f9
Parents: 7e8fc2b
Author: Sergey Shelukhin <se...@apache.org>
Authored: Thu Nov 19 15:02:29 2015 -0800
Committer: Owen O'Malley <om...@apache.org>
Committed: Fri Nov 20 13:46:22 2015 -0800
----------------------------------------------------------------------
.../java/org/apache/hive/beeline/BeeLine.java | 13 ++++++-
.../org/apache/hadoop/hive/cli/CliDriver.java | 8 ++++
.../org/apache/hadoop/hive/conf/HiveConf.java | 29 ++++++++++++++-
.../java/org/apache/hadoop/hive/ql/Driver.java | 20 +++++++++-
.../ql/processors/CommandProcessorResponse.java | 15 ++++++--
.../hadoop/hive/ql/processors/SetProcessor.java | 39 ++++++++++++++------
.../hadoop/hive/ql/session/OperationLog.java | 10 +++++
.../cli/operation/HiveCommandOperation.java | 11 ++++++
8 files changed, 127 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/82705f04/beeline/src/java/org/apache/hive/beeline/BeeLine.java
----------------------------------------------------------------------
diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLine.java b/beeline/src/java/org/apache/hive/beeline/BeeLine.java
index 377703f..1e289ca 100644
--- a/beeline/src/java/org/apache/hive/beeline/BeeLine.java
+++ b/beeline/src/java/org/apache/hive/beeline/BeeLine.java
@@ -109,6 +109,7 @@ import org.apache.hive.beeline.cli.CliOptionsProcessor;
* </ul>
*
*/
+@SuppressWarnings("static-access")
public class BeeLine implements Closeable {
private static final ResourceBundle resourceBundle =
ResourceBundle.getBundle(BeeLine.class.getSimpleName());
@@ -657,7 +658,7 @@ public class BeeLine implements Closeable {
Properties confProps = commandLine.getOptionProperties("hiveconf");
for (String propKey : confProps.stringPropertyNames()) {
- getOpts().getHiveConfVariables().put(propKey, confProps.getProperty(propKey));
+ setHiveConfVar(propKey, confProps.getProperty(propKey));
}
Properties hiveVars = commandLine.getOptionProperties("define");
@@ -739,7 +740,7 @@ public class BeeLine implements Closeable {
Properties hiveConfs = cl.getOptionProperties("hiveconf");
for (String key : hiveConfs.stringPropertyNames()) {
- getOpts().getHiveConfVariables().put(key, hiveConfs.getProperty(key));
+ setHiveConfVar(key, hiveConfs.getProperty(key));
}
driver = cl.getOptionValue("d");
@@ -794,6 +795,14 @@ public class BeeLine implements Closeable {
return code;
}
+
+ private void setHiveConfVar(String key, String val) {
+ getOpts().getHiveConfVariables().put(key, val);
+ if (HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname.equals(key) && "mr".equals(val)) {
+ info(HiveConf.generateMrDeprecationWarning());
+ }
+ }
+
private String constructCmd(String url, String user, String pass, String driver, boolean stripPasswd) {
String com = "!connect "
+ url + " "
http://git-wip-us.apache.org/repos/asf/hive/blob/82705f04/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java
----------------------------------------------------------------------
diff --git a/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java b/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java
index b359850..e04f247 100644
--- a/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java
+++ b/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java
@@ -290,6 +290,11 @@ public class CliDriver {
ss.out.println("Query returned non-zero code: " + res.getResponseCode() +
", cause: " + res.getErrorMessage());
}
+ if (res.getConsoleMessages() != null) {
+ for (String consoleMsg : res.getConsoleMessages()) {
+ console.printInfo(consoleMsg);
+ }
+ }
ret = res.getResponseCode();
}
}
@@ -748,6 +753,9 @@ public class CliDriver {
System.err.println("Could not open input file for reading. (" + e.getMessage() + ")");
return 3;
}
+ if ("mr".equals(HiveConf.getVar(conf, ConfVars.HIVE_EXECUTION_ENGINE))) {
+ console.printInfo(HiveConf.generateMrDeprecationWarning());
+ }
setupConsoleReader();
http://git-wip-us.apache.org/repos/asf/hive/blob/82705f04/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 2185f85..0afb964 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2164,7 +2164,9 @@ public class HiveConf extends Configuration {
"Whether to show the unquoted partition names in query results."),
HIVE_EXECUTION_ENGINE("hive.execution.engine", "mr", new StringSet("mr", "tez", "spark"),
- "Chooses execution engine. Options are: mr (Map reduce, default), tez (hadoop 2 only), spark"),
+ "Chooses execution engine. Options are: mr (Map reduce, default), tez, spark. While MR\n" +
+ "remains the default engine for historical reasons, it is itself a historical engine\n" +
+ "and is deprecated in Hive 2 line. It may be removed without further warning."),
HIVE_EXECUTION_MODE("hive.execution.mode", "container", new StringSet("container", "llap"),
"Chooses whether query fragments will run in container or in llap"),
@@ -2571,6 +2573,13 @@ public class HiveConf extends Configuration {
return defaultExpr;
}
+ private Set<String> getValidStringValues() {
+ if (validator == null || !(validator instanceof StringSet)) {
+ throw new RuntimeException(varname + " does not specify a list of valid values");
+ }
+ return ((StringSet)validator).getExpected();
+ }
+
enum VarType {
STRING {
@Override
@@ -3394,4 +3403,22 @@ public class HiveConf extends Configuration {
public static void setLoadHiveServer2Config(boolean loadHiveServer2Config) {
HiveConf.loadHiveServer2Config = loadHiveServer2Config;
}
+
+ public static String getNonMrEngines() {
+ String result = "";
+ for (String s : ConfVars.HIVE_EXECUTION_ENGINE.getValidStringValues()) {
+ if ("mr".equals(s)) continue;
+ if (!result.isEmpty()) {
+ result += ", ";
+ }
+ result += s;
+ }
+ return result;
+ }
+
+ public static String generateMrDeprecationWarning() {
+ return "Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. "
+ + "Consider using a different execution engine (i.e. " + HiveConf.getNonMrEngines()
+ + ") or using Hive 1.X releases.";
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/82705f04/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index fc5a951..fbde9eb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -114,6 +114,7 @@ import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObje
import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject.HivePrivObjectActionType;
import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject.HivePrivilegeObjectType;
import org.apache.hadoop.hive.ql.session.OperationLog;
+import org.apache.hadoop.hive.ql.session.OperationLog.LoggingLevel;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
import org.apache.hadoop.hive.serde2.ByteStream;
@@ -1505,10 +1506,12 @@ public class Driver implements CommandProcessor {
}
}
- int jobs = Utilities.getMRTasks(plan.getRootTasks()).size()
+ int mrJobs = Utilities.getMRTasks(plan.getRootTasks()).size();
+ int jobs = mrJobs
+ Utilities.getTezTasks(plan.getRootTasks()).size()
+ Utilities.getSparkTasks(plan.getRootTasks()).size();
if (jobs > 0) {
+ logMrWarning(mrJobs);
console.printInfo("Query ID = " + plan.getQueryId());
console.printInfo("Total jobs = " + jobs);
}
@@ -1730,6 +1733,21 @@ public class Driver implements CommandProcessor {
return (0);
}
+ private void logMrWarning(int mrJobs) {
+ if (mrJobs <= 0 || !("mr".equals(HiveConf.getVar(conf, ConfVars.HIVE_EXECUTION_ENGINE)))) {
+ return;
+ }
+ String warning = HiveConf.generateMrDeprecationWarning();
+ LOG.warn(warning);
+ warning = "WARNING: " + warning;
+ console.printInfo(warning);
+ // Propagate warning to beeline via operation log.
+ OperationLog ol = OperationLog.getCurrentOperationLog();
+ if (ol != null) {
+ ol.writeOperationLog(LoggingLevel.EXECUTION, warning + "\n");
+ }
+ }
+
private void setErrorMsgAndDetail(int exitVal, Throwable downstreamError, Task tsk) {
this.downstreamError = downstreamError;
errorMessage = "FAILED: Execution Error, return code " + exitVal + " from " + tsk.getClass().getName();
http://git-wip-us.apache.org/repos/asf/hive/blob/82705f04/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorResponse.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorResponse.java b/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorResponse.java
index 21b7457..d4f35f1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorResponse.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorResponse.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.hive.ql.processors;
+import java.util.List;
+
import org.apache.hadoop.hive.metastore.api.Schema;
import org.apache.hadoop.hive.ql.ErrorMsg;
@@ -36,6 +38,7 @@ public class CommandProcessorResponse {
private final Schema resSchema;
private final Throwable exception;
+ private final List<String> consoleMessages;
public CommandProcessorResponse(int responseCode) {
this(responseCode, null, null, null, null);
@@ -45,6 +48,10 @@ public class CommandProcessorResponse {
this(responseCode, errorMessage, SQLState, null, null);
}
+ public CommandProcessorResponse(int responseCode, List<String> consoleMessages) {
+ this(responseCode, null, null, null, null, -1, consoleMessages);
+ }
+
public CommandProcessorResponse(int responseCode, String errorMessage, String SQLState, Throwable exception) {
this(responseCode, errorMessage, SQLState, null, exception);
}
@@ -54,7 +61,7 @@ public class CommandProcessorResponse {
}
public CommandProcessorResponse(int responseCode, ErrorMsg canonicalErrMsg, Throwable t, String ... msgArgs) {
this(responseCode, canonicalErrMsg.format(msgArgs),
- canonicalErrMsg.getSQLState(), null, t, canonicalErrMsg.getErrorCode());
+ canonicalErrMsg.getSQLState(), null, t, canonicalErrMsg.getErrorCode(), null);
}
/**
@@ -71,16 +78,17 @@ public class CommandProcessorResponse {
public CommandProcessorResponse(int responseCode, String errorMessage, String SQLState,
Schema schema, Throwable exception) {
- this(responseCode, errorMessage, SQLState, schema, exception, -1);
+ this(responseCode, errorMessage, SQLState, schema, exception, -1, null);
}
public CommandProcessorResponse(int responseCode, String errorMessage, String SQLState,
- Schema schema, Throwable exception, int hiveErrorCode) {
+ Schema schema, Throwable exception, int hiveErrorCode, List<String> consoleMessages) {
this.responseCode = responseCode;
this.errorMessage = errorMessage;
this.SQLState = SQLState;
this.resSchema = schema;
this.exception = exception;
this.hiveErrorCode = hiveErrorCode;
+ this.consoleMessages = consoleMessages;
}
public int getResponseCode() { return responseCode; }
@@ -89,6 +97,7 @@ public class CommandProcessorResponse {
public Schema getSchema() { return resSchema; }
public Throwable getException() { return exception; }
public int getErrorCode() { return hiveErrorCode; }
+ public List<String> getConsoleMessages() { return consoleMessages; }
public String toString() {
return "(" + responseCode + "," + errorMessage + "," +
(hiveErrorCode > 0 ? hiveErrorCode + "," : "" ) +
http://git-wip-us.apache.org/repos/asf/hive/blob/82705f04/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java
index 9389759..9a3ba04 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java
@@ -37,12 +37,17 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Schema;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.session.SessionState;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Lists;
/**
* SetProcessor.
*
*/
public class SetProcessor implements CommandProcessor {
+ private static final Logger LOG = LoggerFactory.getLogger(SetProcessor.class);
private static final String prefix = "set: ";
@@ -110,22 +115,24 @@ public class SetProcessor implements CommandProcessor {
public CommandProcessorResponse executeSetVariable(String varname, String varvalue) {
try {
- return new CommandProcessorResponse(setVariable(varname, varvalue));
+ return setVariable(varname, varvalue);
} catch (Exception e) {
return new CommandProcessorResponse(1, e.getMessage(), "42000",
e instanceof IllegalArgumentException ? null : e);
}
}
- public static int setVariable(String varname, String varvalue) throws Exception {
+ public static CommandProcessorResponse setVariable(
+ String varname, String varvalue) throws Exception {
SessionState ss = SessionState.get();
if (varvalue.contains("\n")){
ss.err.println("Warning: Value had a \\n character in it.");
}
varname = varname.trim();
+ String nonErrorMessage = null;
if (varname.startsWith(ENV_PREFIX)){
ss.err.println("env:* variables can not be set.");
- return 1;
+ return new CommandProcessorResponse(1); // Should we propagate the error message properly?
} else if (varname.startsWith(SYSTEM_PREFIX)){
String propName = varname.substring(SYSTEM_PREFIX.length());
System.getProperties()
@@ -137,7 +144,7 @@ public class SetProcessor implements CommandProcessor {
}).substitute(ss.getConf(), varvalue));
} else if (varname.startsWith(HIVECONF_PREFIX)){
String propName = varname.substring(HIVECONF_PREFIX.length());
- setConf(varname, propName, varvalue, false);
+ nonErrorMessage = setConf(varname, propName, varvalue, false);
} else if (varname.startsWith(HIVEVAR_PREFIX)) {
String propName = varname.substring(HIVEVAR_PREFIX.length());
ss.getHiveVariables().put(propName, new VariableSubstitution(new HiveVariableSource() {
@@ -156,17 +163,21 @@ public class SetProcessor implements CommandProcessor {
}
}).substitute(ss.getConf(), varvalue));
} else {
- setConf(varname, varname, varvalue, true);
+ nonErrorMessage = setConf(varname, varname, varvalue, true);
if (varname.equals(HiveConf.ConfVars.HIVE_SESSION_HISTORY_ENABLED.toString())) {
SessionState.get().updateHistory(Boolean.parseBoolean(varvalue), ss);
}
}
- return 0;
+ return nonErrorMessage == null ? new CommandProcessorResponse(0)
+ : new CommandProcessorResponse(0, Lists.newArrayList(nonErrorMessage));
}
- // returns non-null string for validation fail
- private static void setConf(String varname, String key, String varvalue, boolean register)
+ /**
+ * @return A console message that is not strong enough to fail the command (e.g. deprecation).
+ */
+ private static String setConf(String varname, String key, String varvalue, boolean register)
throws IllegalArgumentException {
+ String result = null;
HiveConf conf = SessionState.get().getConf();
String value = new VariableSubstitution(new HiveVariableSource() {
@Override
@@ -196,13 +207,19 @@ public class SetProcessor implements CommandProcessor {
}
}
conf.verifyAndSet(key, value);
- if (HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname.equals(key)
- && !"spark".equals(value)) {
- SessionState.get().closeSparkSession();
+ if (HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname.equals(key)) {
+ if (!"spark".equals(value)) {
+ SessionState.get().closeSparkSession();
+ }
+ if ("mr".equals(value)) {
+ result = HiveConf.generateMrDeprecationWarning();
+ LOG.warn(result);
+ }
}
if (register) {
SessionState.get().getOverriddenConfigurations().put(key, value);
}
+ return result;
}
private SortedMap<String,String> propertiesToSortedMap(Properties p){
http://git-wip-us.apache.org/repos/asf/hive/blob/82705f04/ql/src/java/org/apache/hadoop/hive/ql/session/OperationLog.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/OperationLog.java b/ql/src/java/org/apache/hadoop/hive/ql/session/OperationLog.java
index fb7dda5..2ecdde9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/OperationLog.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/OperationLog.java
@@ -103,6 +103,16 @@ public class OperationLog {
}
/**
+ * Write operation execution logs into log file
+ * @param operationLogMessage one line of log emitted from log4j
+ */
+ public void writeOperationLog(LoggingLevel level, String operationLogMessage) {
+ if (opLoggingLevel.compareTo(level) < 0) return;
+ logFile.write(operationLogMessage);
+ }
+
+
+ /**
* Read operation execution logs from log file
* @param isFetchFirst true if the Enum FetchOrientation value is Fetch_First
* @param maxRows the max number of fetched lines from log
http://git-wip-us.apache.org/repos/asf/hive/blob/82705f04/service/src/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java b/service/src/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java
index 22c55f1..04dc6e3 100644
--- a/service/src/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java
+++ b/service/src/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java
@@ -34,7 +34,9 @@ import org.apache.commons.lang3.CharEncoding;
import org.apache.hadoop.hive.metastore.api.Schema;
import org.apache.hadoop.hive.ql.processors.CommandProcessor;
import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
+import org.apache.hadoop.hive.ql.session.OperationLog;
import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.ql.session.OperationLog.LoggingLevel;
import org.apache.hive.service.ServiceUtils;
import org.apache.hive.service.cli.FetchOrientation;
import org.apache.hive.service.cli.HiveSQLException;
@@ -123,6 +125,15 @@ public class HiveCommandOperation extends ExecuteStatementOperation {
setHasResultSet(false);
resultSchema = new TableSchema();
}
+ if (response.getConsoleMessages() != null) {
+ // Propagate processor messages (if any) to beeline or other client.
+ OperationLog ol = OperationLog.getCurrentOperationLog();
+ if (ol != null) {
+ for (String consoleMsg : response.getConsoleMessages()) {
+ ol.writeOperationLog(LoggingLevel.EXECUTION, consoleMsg + "\n");
+ }
+ }
+ }
} catch (HiveSQLException e) {
setState(OperationState.ERROR);
throw e;
[07/12] hive git commit: HIVE-11684: Implement limit pushdown through
outer join in CBO (Jesus Camacho Rodriguez,
reviewed by Laljo John Pullokkaran)
Posted by om...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/39bd58b2/ql/src/test/results/clientpositive/spark/join32_lessSize.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/join32_lessSize.q.out b/ql/src/test/results/clientpositive/spark/join32_lessSize.q.out
index 937e8fc..e2ae796 100644
--- a/ql/src/test/results/clientpositive/spark/join32_lessSize.q.out
+++ b/ql/src/test/results/clientpositive/spark/join32_lessSize.q.out
@@ -1767,18 +1767,22 @@ STAGE PLANS:
Map 1
Map Operator Tree:
TableScan
- alias: y
+ alias: z
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
GatherStats: false
- Select Operator
- expressions: key (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Spark HashTable Sink Operator
- keys:
- 0 _col0 (type: string)
- 1 _col0 (type: string)
- Position of Big Table: 1
+ Filter Operator
+ isSamplingPred: false
+ predicate: value is not null (type: boolean)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: value (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Spark HashTable Sink Operator
+ keys:
+ 0 _col0 (type: string)
+ 1 _col1 (type: string)
+ Position of Big Table: 1
Local Work:
Map Reduce Local Work
Path -> Alias:
@@ -1786,9 +1790,12 @@ STAGE PLANS:
Path -> Partition:
#### A masked pattern was here ####
Partition
- base file name: src
+ base file name: hr=11
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ partition values:
+ ds 2008-04-08
+ hr 11
properties:
COLUMN_STATS_ACCURATE true
bucket_count -1
@@ -1796,11 +1803,13 @@ STAGE PLANS:
columns.comments 'default','default'
columns.types string:string
#### A masked pattern was here ####
- name default.src
+ name default.srcpart
numFiles 1
numRows 500
+ partition_columns ds/hr
+ partition_columns.types string:string
rawDataSize 5312
- serialization.ddl struct src { string key, string value}
+ serialization.ddl struct srcpart { string key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
totalSize 5812
@@ -1810,45 +1819,38 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE true
bucket_count -1
columns key,value
columns.comments 'default','default'
columns.types string:string
#### A masked pattern was here ####
- name default.src
- numFiles 1
- numRows 500
- rawDataSize 5312
- serialization.ddl struct src { string key, string value}
+ name default.srcpart
+ partition_columns ds/hr
+ partition_columns.types string:string
+ serialization.ddl struct srcpart { string key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- totalSize 5812
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.src
- name: default.src
+ name: default.srcpart
+ name: default.srcpart
Truncated Path -> Alias:
- /src [y]
- Map 3
+ /srcpart/ds=2008-04-08/hr=11 [z]
+ Map 2
Map Operator Tree:
TableScan
- alias: z
+ alias: y
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
GatherStats: false
- Filter Operator
- isSamplingPred: false
- predicate: value is not null (type: boolean)
- Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: value (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Spark HashTable Sink Operator
- keys:
- 0 _col1 (type: string)
- 1 _col0 (type: string)
- Position of Big Table: 0
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Spark HashTable Sink Operator
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ Position of Big Table: 1
Local Work:
Map Reduce Local Work
Path -> Alias:
@@ -1856,12 +1858,9 @@ STAGE PLANS:
Path -> Partition:
#### A masked pattern was here ####
Partition
- base file name: hr=11
+ base file name: src
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- partition values:
- ds 2008-04-08
- hr 11
properties:
COLUMN_STATS_ACCURATE true
bucket_count -1
@@ -1869,13 +1868,11 @@ STAGE PLANS:
columns.comments 'default','default'
columns.types string:string
#### A masked pattern was here ####
- name default.srcpart
+ name default.src
numFiles 1
numRows 500
- partition_columns ds/hr
- partition_columns.types string:string
rawDataSize 5312
- serialization.ddl struct srcpart { string key, string value}
+ serialization.ddl struct src { string key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
totalSize 5812
@@ -1885,29 +1882,32 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
+ COLUMN_STATS_ACCURATE true
bucket_count -1
columns key,value
columns.comments 'default','default'
columns.types string:string
#### A masked pattern was here ####
- name default.srcpart
- partition_columns ds/hr
- partition_columns.types string:string
- serialization.ddl struct srcpart { string key, string value}
+ name default.src
+ numFiles 1
+ numRows 500
+ rawDataSize 5312
+ serialization.ddl struct src { string key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5812
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.srcpart
- name: default.srcpart
+ name: default.src
+ name: default.src
Truncated Path -> Alias:
- /srcpart/ds=2008-04-08/hr=11 [z]
+ /src [y]
Stage: Stage-1
Spark
#### A masked pattern was here ####
Vertices:
- Map 2
+ Map 3
Map Operator Tree:
TableScan
alias: x
@@ -1929,7 +1929,7 @@ STAGE PLANS:
1 _col0 (type: string)
outputColumnNames: _col1, _col2
input vertices:
- 0 Map 1
+ 0 Map 2
Position of Big Table: 1
Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
Select Operator
@@ -1940,15 +1940,15 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col1 (type: string)
- 1 _col0 (type: string)
- outputColumnNames: _col0, _col1, _col2
+ 0 _col0 (type: string)
+ 1 _col1 (type: string)
+ outputColumnNames: _col0, _col3, _col4
input vertices:
- 1 Map 3
- Position of Big Table: 0
+ 0 Map 1
+ Position of Big Table: 1
Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: string), _col2 (type: string), _col1 (type: string)
+ expressions: _col3 (type: string), _col0 (type: string), _col4 (type: string)
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE
File Output Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/39bd58b2/ql/src/test/results/clientpositive/spark/join_merging.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/join_merging.q.out b/ql/src/test/results/clientpositive/spark/join_merging.q.out
index 6c5089b..baed227 100644
--- a/ql/src/test/results/clientpositive/spark/join_merging.q.out
+++ b/ql/src/test/results/clientpositive/spark/join_merging.q.out
@@ -114,8 +114,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
- Reducer 3 <- Map 5 (PARTITION-LEVEL SORT, 2), Reducer 2 (PARTITION-LEVEL SORT, 2)
+ Reducer 4 <- Map 3 (PARTITION-LEVEL SORT, 2), Map 5 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Reducer 4 (PARTITION-LEVEL SORT, 2)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -123,6 +123,20 @@ STAGE PLANS:
TableScan
alias: p1
Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: p_partkey (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: p1
+ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (p_size > 10) (type: boolean)
Statistics: Num rows: 8 Data size: 968 Basic stats: COMPLETE Column stats: NONE
@@ -136,7 +150,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 8 Data size: 968 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: int)
- Map 4
+ Map 5
Map Operator Tree:
TableScan
alias: p1
@@ -151,20 +165,6 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: int)
- Map 5
- Map Operator Tree:
- TableScan
- alias: p1
- Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: p_partkey (type: int)
- outputColumnNames: _col0
- Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
Reducer 2
Reduce Operator Tree:
Join Operator
@@ -172,6 +172,27 @@ STAGE PLANS:
Left Outer Join0 to 1
keys:
0 _col0 (type: int)
+ 1 _col2 (type: int)
+ outputColumnNames: _col2, _col4
+ Statistics: Num rows: 28 Data size: 3461 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col2 (type: int), _col4 (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 28 Data size: 3461 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 28 Data size: 3461 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Reducer 4
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ keys:
+ 0 _col0 (type: int)
1 _col0 (type: int)
outputColumnNames: _col1, _col2, _col3
Statistics: Num rows: 28 Data size: 3461 Basic stats: COMPLETE Column stats: NONE
@@ -184,27 +205,6 @@ STAGE PLANS:
Map-reduce partition columns: _col2 (type: int)
Statistics: Num rows: 9 Data size: 1112 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: int), _col3 (type: int)
- Reducer 3
- Reduce Operator Tree:
- Join Operator
- condition map:
- Right Outer Join0 to 1
- keys:
- 0 _col2 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col1, _col3
- Statistics: Num rows: 28 Data size: 3461 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col1 (type: int), _col3 (type: int)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 28 Data size: 3461 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 28 Data size: 3461 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/39bd58b2/ql/src/test/results/clientpositive/subquery_notin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/subquery_notin.q.out b/ql/src/test/results/clientpositive/subquery_notin.q.out
index 5563794..12bd35e 100644
--- a/ql/src/test/results/clientpositive/subquery_notin.q.out
+++ b/ql/src/test/results/clientpositive/subquery_notin.q.out
@@ -1,4 +1,4 @@
-Warning: Shuffle Join JOIN[20][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
+Warning: Shuffle Join JOIN[23][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product
PREHOOK: query: -- non agg, non corr
explain
select *
@@ -18,64 +18,70 @@ where src.key not in
)
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-3 is a root stage
- Stage-1 depends on stages: Stage-3
- Stage-2 depends on stages: Stage-1
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1, Stage-4
+ Stage-4 is a root stage
Stage-0 depends on stages: Stage-2
STAGE PLANS:
- Stage: Stage-3
+ Stage: Stage-1
Map Reduce
Map Operator Tree:
TableScan
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((key > '2') and key is null) (type: boolean)
- Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+ predicate: (key > '2') (type: boolean)
+ Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
Select Operator
- Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: count()
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: bigint)
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
Reduce Operator Tree:
- Group By Operator
- aggregations: count(VALUE._col0)
- mode: mergepartial
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (_col0 = 0) (type: boolean)
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+ predicate: _col2 is null (type: boolean)
+ Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
- Stage: Stage-1
+ Stage: Stage-2
Map Reduce
Map Operator Tree:
TableScan
- alias: src
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: key (type: string), value (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: string), _col1 (type: string)
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string), _col1 (type: string)
TableScan
Reduce Output Operator
sort order:
@@ -88,62 +94,52 @@ STAGE PLANS:
0
1
outputColumnNames: _col0, _col1
- Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE
table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- Stage: Stage-2
+ Stage: Stage-4
Map Reduce
Map Operator Tree:
TableScan
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: string)
- TableScan
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key > '2') (type: boolean)
- Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((key > '2') and key is null) (type: boolean)
+ Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: key (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
Reduce Operator Tree:
- Join Operator
- condition map:
- Left Outer Join0 to 1
- keys:
- 0 _col0 (type: string)
- 1 _col0 (type: string)
- outputColumnNames: _col0, _col1, _col3
- Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: _col3 is null (type: boolean)
- Statistics: Num rows: 302 Data size: 3208 Basic stats: COMPLETE Column stats: NONE
+ predicate: (_col0 = 0) (type: boolean)
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: string), _col1 (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 302 Data size: 3208 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 302 Data size: 3208 Basic stats: COMPLETE Column stats: NONE
table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
Stage: Stage-0
Fetch Operator
@@ -151,7 +147,7 @@ STAGE PLANS:
Processor Tree:
ListSink
-Warning: Shuffle Join JOIN[20][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
+Warning: Shuffle Join JOIN[23][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product
PREHOOK: query: select *
from src
where src.key not in ( select key from src s1 where s1.key > '2')
@@ -1243,7 +1239,7 @@ Manufacturer#5 almond antique medium spring khaki 6
Manufacturer#5 almond azure blanched chiffon midnight 23
Manufacturer#5 almond antique blue firebrick mint 31
Manufacturer#5 almond aquamarine dodger light gainsboro 46
-Warning: Shuffle Join JOIN[20][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
+Warning: Shuffle Join JOIN[23][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product
PREHOOK: query: -- non agg, non corr, Group By in Parent Query
select li.l_partkey, count(*)
from lineitem li
@@ -1278,7 +1274,7 @@ POSTHOOK: Input: default@lineitem
139636 1
175839 1
182052 1
-Warning: Shuffle Join JOIN[20][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
+Warning: Shuffle Join JOIN[23][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product
PREHOOK: query: -- alternate not in syntax
select *
from src
@@ -1442,7 +1438,7 @@ POSTHOOK: Input: default@src
POSTHOOK: Input: default@t1_v
POSTHOOK: Output: database:default
POSTHOOK: Output: default@T2_v
-Warning: Shuffle Join JOIN[24][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
+Warning: Shuffle Join JOIN[27][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product
PREHOOK: query: explain
select *
from T1_v where T1_v.key not in (select T2_v.key from T2_v)
@@ -1452,50 +1448,12 @@ select *
from T1_v where T1_v.key not in (select T2_v.key from T2_v)
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-3 is a root stage
- Stage-1 depends on stages: Stage-3
- Stage-2 depends on stages: Stage-1
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1, Stage-4
+ Stage-4 is a root stage
Stage-0 depends on stages: Stage-2
STAGE PLANS:
- Stage: Stage-3
- Map Reduce
- Map Operator Tree:
- TableScan
- alias: src
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: ((key < '11') and CASE WHEN ((key > '104')) THEN (null) ELSE (key) END is null) (type: boolean)
- Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: count()
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: bigint)
- Reduce Operator Tree:
- Group By Operator
- aggregations: count(VALUE._col0)
- mode: mergepartial
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: (_col0 = 0) (type: boolean)
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
-
Stage: Stage-1
Map Reduce
Map Operator Tree:
@@ -1510,9 +1468,52 @@ STAGE PLANS:
outputColumnNames: _col0
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- sort order:
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: string)
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((key < '11') and (CASE WHEN ((key > '104')) THEN (null) ELSE (key) END < '11')) (type: boolean)
+ Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: CASE WHEN ((key > '104')) THEN (null) ELSE (key) END (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: _col1 is null (type: boolean)
+ Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
TableScan
Reduce Output Operator
sort order:
@@ -1525,61 +1526,52 @@ STAGE PLANS:
0
1
outputColumnNames: _col0
- Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 100 Data size: 1065 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ Statistics: Num rows: 100 Data size: 1065 Basic stats: COMPLETE Column stats: NONE
table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- Stage: Stage-2
+ Stage: Stage-4
Map Reduce
Map Operator Tree:
TableScan
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE
- TableScan
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((key < '11') and (CASE WHEN ((key > '104')) THEN (null) ELSE (key) END < '11')) (type: boolean)
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((key < '11') and CASE WHEN ((key > '104')) THEN (null) ELSE (key) END is null) (type: boolean)
+ Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: CASE WHEN ((key > '104')) THEN (null) ELSE (key) END (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
Reduce Operator Tree:
- Join Operator
- condition map:
- Left Outer Join0 to 1
- keys:
- 0 _col0 (type: string)
- 1 _col0 (type: string)
- outputColumnNames: _col0, _col2
- Statistics: Num rows: 200 Data size: 2132 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: _col2 is null (type: boolean)
- Statistics: Num rows: 100 Data size: 1066 Basic stats: COMPLETE Column stats: NONE
+ predicate: (_col0 = 0) (type: boolean)
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 100 Data size: 1066 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 100 Data size: 1066 Basic stats: COMPLETE Column stats: NONE
table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
Stage: Stage-0
Fetch Operator
@@ -1587,7 +1579,7 @@ STAGE PLANS:
Processor Tree:
ListSink
-Warning: Shuffle Join JOIN[24][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
+Warning: Shuffle Join JOIN[27][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product
PREHOOK: query: select *
from T1_v where T1_v.key not in (select T2_v.key from T2_v)
PREHOOK: type: QUERY
http://git-wip-us.apache.org/repos/asf/hive/blob/39bd58b2/ql/src/test/results/clientpositive/subquery_notin_having.q.java1.7.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/subquery_notin_having.q.java1.7.out b/ql/src/test/results/clientpositive/subquery_notin_having.q.java1.7.out
index 9689ae3..150b527 100644
--- a/ql/src/test/results/clientpositive/subquery_notin_having.q.java1.7.out
+++ b/ql/src/test/results/clientpositive/subquery_notin_having.q.java1.7.out
@@ -1,4 +1,4 @@
-Warning: Shuffle Join JOIN[24][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-2:MAPRED' is a cross product
+Warning: Shuffle Join JOIN[27][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-3:MAPRED' is a cross product
PREHOOK: query: -- non agg, non corr
-- JAVA_VERSION_SPECIFIC_OUTPUT
@@ -25,9 +25,9 @@ having key not in
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-1 is a root stage
- Stage-2 depends on stages: Stage-1, Stage-4
- Stage-3 depends on stages: Stage-2
- Stage-4 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-3 depends on stages: Stage-2, Stage-5
+ Stage-5 is a root stage
Stage-0 depends on stages: Stage-3
STAGE PLANS:
@@ -72,38 +72,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
Reduce Output Operator
- sort order:
- Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: string), _col1 (type: bigint)
- TableScan
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0
- 1
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
-
- Stage: Stage-3
- Map Reduce
- Map Operator Tree:
- TableScan
- Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: bigint)
TableScan
alias: src
@@ -127,24 +99,48 @@ STAGE PLANS:
keys:
0 _col0 (type: string)
1 _col0 (type: string)
- outputColumnNames: _col0, _col1, _col3
- Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: _col3 is null (type: boolean)
- Statistics: Num rows: 151 Data size: 1606 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: bigint)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 151 Data size: 1606 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 151 Data size: 1606 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ predicate: _col2 is null (type: boolean)
+ Statistics: Num rows: 137 Data size: 1455 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
- Stage: Stage-4
+ Stage: Stage-3
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 137 Data size: 1455 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string), _col1 (type: bigint)
+ TableScan
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0
+ 1
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 150 Data size: 1600 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 150 Data size: 1600 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-5
Map Reduce
Map Operator Tree:
TableScan
http://git-wip-us.apache.org/repos/asf/hive/blob/39bd58b2/ql/src/test/results/clientpositive/tez/explainuser_1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/explainuser_1.q.out b/ql/src/test/results/clientpositive/tez/explainuser_1.q.out
index fa29dfe..871fa4e 100644
--- a/ql/src/test/results/clientpositive/tez/explainuser_1.q.out
+++ b/ql/src/test/results/clientpositive/tez/explainuser_1.q.out
@@ -3732,108 +3732,105 @@ POSTHOOK: type: QUERY
Plan optimized by CBO.
Vertex dependency in root stage
-Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE)
-Reducer 3 <- Map 7 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
+Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE)
+Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE)
Reducer 4 <- Reducer 3 (SIMPLE_EDGE)
-Reducer 6 <- Map 5 (SIMPLE_EDGE)
+Reducer 7 <- Map 6 (SIMPLE_EDGE)
Stage-0
Fetch Operator
limit:-1
Stage-1
Reducer 4
- File Output Operator [FS_28]
+ File Output Operator [FS_29]
compressed:false
- Statistics:Num rows: 302 Data size: 53756 Basic stats: COMPLETE Column stats: NONE
+ Statistics:Num rows: 1 Data size: 291 Basic stats: COMPLETE Column stats: NONE
table:{"input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"}
- Select Operator [SEL_27]
+ Select Operator [SEL_28]
| outputColumnNames:["_col0","_col1"]
- | Statistics:Num rows: 302 Data size: 53756 Basic stats: COMPLETE Column stats: NONE
+ | Statistics:Num rows: 1 Data size: 291 Basic stats: COMPLETE Column stats: NONE
|<-Reducer 3 [SIMPLE_EDGE]
- Reduce Output Operator [RS_26]
+ Reduce Output Operator [RS_27]
key expressions:_col0 (type: string)
sort order:+
- Statistics:Num rows: 302 Data size: 53756 Basic stats: COMPLETE Column stats: NONE
+ Statistics:Num rows: 1 Data size: 291 Basic stats: COMPLETE Column stats: NONE
value expressions:_col1 (type: string)
- Select Operator [SEL_25]
- outputColumnNames:["_col0","_col1"]
- Statistics:Num rows: 302 Data size: 53756 Basic stats: COMPLETE Column stats: NONE
- Filter Operator [FIL_31]
- predicate:_col3 is null (type: boolean)
- Statistics:Num rows: 302 Data size: 53756 Basic stats: COMPLETE Column stats: NONE
- Merge Join Operator [MERGEJOIN_36]
- | condition map:[{"":"Left Outer Join0 to 1"}]
- | keys:{"0":"_col0 (type: string)","1":"_col0 (type: string)"}
- | outputColumnNames:["_col0","_col1","_col3"]
- | Statistics:Num rows: 605 Data size: 107690 Basic stats: COMPLETE Column stats: NONE
- |<-Map 7 [SIMPLE_EDGE]
- | Reduce Output Operator [RS_22]
- | key expressions:_col0 (type: string)
- | Map-reduce partition columns:_col0 (type: string)
- | sort order:+
- | Statistics:Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE
- | Select Operator [SEL_16]
- | outputColumnNames:["_col0"]
- | Statistics:Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE
- | Filter Operator [FIL_34]
- | predicate:(key > '2') (type: boolean)
- | Statistics:Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE
- | TableScan [TS_14]
- | alias:src_cbo
- | Statistics:Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
- |<-Reducer 2 [SIMPLE_EDGE]
- Reduce Output Operator [RS_21]
- key expressions:_col0 (type: string)
- Map-reduce partition columns:_col0 (type: string)
- sort order:+
- Statistics:Num rows: 550 Data size: 97900 Basic stats: COMPLETE Column stats: NONE
- value expressions:_col1 (type: string)
- Merge Join Operator [MERGEJOIN_35]
- | condition map:[{"":"Inner Join 0 to 1"}]
- | keys:{}
- | outputColumnNames:["_col0","_col1"]
- | Statistics:Num rows: 550 Data size: 97900 Basic stats: COMPLETE Column stats: NONE
- |<-Map 1 [SIMPLE_EDGE]
- | Reduce Output Operator [RS_18]
- | sort order:
- | Statistics:Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
- | value expressions:_col0 (type: string), _col1 (type: string)
- | Select Operator [SEL_2]
- | outputColumnNames:["_col0","_col1"]
- | Statistics:Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
- | TableScan [TS_0]
- | alias:src_cbo
- | Statistics:Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
- |<-Reducer 6 [SIMPLE_EDGE]
- Reduce Output Operator [RS_19]
+ Merge Join Operator [MERGEJOIN_37]
+ | condition map:[{"":"Inner Join 0 to 1"}]
+ | keys:{}
+ | outputColumnNames:["_col0","_col1"]
+ | Statistics:Num rows: 1 Data size: 291 Basic stats: COMPLETE Column stats: NONE
+ |<-Reducer 2 [SIMPLE_EDGE]
+ | Reduce Output Operator [RS_21]
+ | sort order:
+ | Statistics:Num rows: 1 Data size: 265 Basic stats: COMPLETE Column stats: COMPLETE
+ | value expressions:_col0 (type: string), _col1 (type: string)
+ | Filter Operator [FIL_32]
+ | predicate:_col2 is null (type: boolean)
+ | Statistics:Num rows: 1 Data size: 265 Basic stats: COMPLETE Column stats: COMPLETE
+ | Merge Join Operator [MERGEJOIN_36]
+ | | condition map:[{"":"Left Outer Join0 to 1"}]
+ | | keys:{"0":"_col0 (type: string)","1":"_col0 (type: string)"}
+ | | outputColumnNames:["_col0","_col1","_col2"]
+ | | Statistics:Num rows: 404 Data size: 107060 Basic stats: COMPLETE Column stats: COMPLETE
+ | |<-Map 1 [SIMPLE_EDGE]
+ | | Reduce Output Operator [RS_18]
+ | | key expressions:_col0 (type: string)
+ | | Map-reduce partition columns:_col0 (type: string)
+ | | sort order:+
+ | | Statistics:Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ | | value expressions:_col1 (type: string)
+ | | Select Operator [SEL_2]
+ | | outputColumnNames:["_col0","_col1"]
+ | | Statistics:Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ | | TableScan [TS_0]
+ | | alias:src_cbo
+ | | Statistics:Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ | |<-Map 5 [SIMPLE_EDGE]
+ | Reduce Output Operator [RS_19]
+ | key expressions:_col0 (type: string)
+ | Map-reduce partition columns:_col0 (type: string)
+ | sort order:+
+ | Statistics:Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE
+ | Select Operator [SEL_5]
+ | outputColumnNames:["_col0"]
+ | Statistics:Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE
+ | Filter Operator [FIL_33]
+ | predicate:(key > '2') (type: boolean)
+ | Statistics:Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE
+ | TableScan [TS_3]
+ | alias:src_cbo
+ | Statistics:Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
+ |<-Reducer 7 [SIMPLE_EDGE]
+ Reduce Output Operator [RS_22]
+ sort order:
+ Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Select Operator [SEL_14]
+ Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Filter Operator [FIL_34]
+ predicate:(_col0 = 0) (type: boolean)
+ Statistics:Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator [GBY_13]
+ | aggregations:["count(VALUE._col0)"]
+ | outputColumnNames:["_col0"]
+ | Statistics:Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ |<-Map 6 [SIMPLE_EDGE]
+ Reduce Output Operator [RS_12]
sort order:
- Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
- Select Operator [SEL_11]
- Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
- Filter Operator [FIL_32]
- predicate:(_col0 = 0) (type: boolean)
- Statistics:Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator [GBY_10]
- | aggregations:["count(VALUE._col0)"]
- | outputColumnNames:["_col0"]
- | Statistics:Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- |<-Map 5 [SIMPLE_EDGE]
- Reduce Output Operator [RS_9]
- sort order:
- Statistics:Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions:_col0 (type: bigint)
- Group By Operator [GBY_8]
- aggregations:["count()"]
- outputColumnNames:["_col0"]
- Statistics:Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator [SEL_5]
- Statistics:Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator [FIL_33]
- predicate:((key > '2') and key is null) (type: boolean)
- Statistics:Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE
- TableScan [TS_3]
- alias:src_cbo
- Statistics:Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics:Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions:_col0 (type: bigint)
+ Group By Operator [GBY_11]
+ aggregations:["count()"]
+ outputColumnNames:["_col0"]
+ Statistics:Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator [SEL_8]
+ Statistics:Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator [FIL_35]
+ predicate:((key > '2') and key is null) (type: boolean)
+ Statistics:Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE
+ TableScan [TS_6]
+ alias:src_cbo
+ Statistics:Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
PREHOOK: query: explain select p_mfgr, b.p_name, p_size
from part b
http://git-wip-us.apache.org/repos/asf/hive/blob/39bd58b2/ql/src/test/results/clientpositive/tez/tez_dynpart_hashjoin_3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/tez_dynpart_hashjoin_3.q.out b/ql/src/test/results/clientpositive/tez/tez_dynpart_hashjoin_3.q.out
new file mode 100644
index 0000000..52b4288
--- /dev/null
+++ b/ql/src/test/results/clientpositive/tez/tez_dynpart_hashjoin_3.q.out
@@ -0,0 +1,200 @@
+PREHOOK: query: explain
+select a.*
+from alltypesorc a left outer join src b
+on a.cint = cast(b.key as int) and (a.cint < 100)
+limit 1
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select a.*
+from alltypesorc a left outer join src b
+on a.cint = cast(b.key as int) and (a.cint < 100)
+limit 1
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE)
+ Reducer 4 <- Map 3 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: b
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: UDFToInteger(_col0) (type: int)
+ sort order: +
+ Map-reduce partition columns: UDFToInteger(_col0) (type: int)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11
+ Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 1
+ Statistics: Num rows: 1 Data size: 215 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 215 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean)
+ Reducer 2
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Right Outer Join0 to 1
+ filter predicates:
+ 0
+ 1 {(KEY.reducesinkkey0 < 100)}
+ keys:
+ 0 UDFToInteger(_col0) (type: int)
+ 1 _col2 (type: int)
+ outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col1 (type: tinyint), _col2 (type: smallint), _col3 (type: int), _col4 (type: bigint), _col5 (type: float), _col6 (type: double), _col7 (type: string), _col8 (type: string), _col9 (type: timestamp), _col10 (type: timestamp), _col11 (type: boolean), _col12 (type: boolean)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 1
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Reducer 4
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: tinyint), VALUE._col1 (type: smallint), VALUE._col2 (type: int), VALUE._col3 (type: bigint), VALUE._col4 (type: float), VALUE._col5 (type: double), VALUE._col6 (type: string), VALUE._col7 (type: string), VALUE._col8 (type: timestamp), VALUE._col9 (type: timestamp), VALUE._col10 (type: boolean), VALUE._col11 (type: boolean)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11
+ Statistics: Num rows: 1 Data size: 215 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 1
+ Statistics: Num rows: 1 Data size: 215 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col2 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col2 (type: int)
+ Statistics: Num rows: 1 Data size: 215 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean)
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain
+select a.*
+from alltypesorc a left outer join src b
+on a.cint = cast(b.key as int) and (a.cint < 100)
+limit 1
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select a.*
+from alltypesorc a left outer join src b
+on a.cint = cast(b.key as int) and (a.cint < 100)
+limit 1
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 3 <- Map 1 (BROADCAST_EDGE), Map 2 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: b
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: UDFToInteger(_col0) (type: int)
+ sort order: +
+ Map-reduce partition columns: UDFToInteger(_col0) (type: int)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Map 2
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11
+ Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 1
+ Statistics: Num rows: 1 Data size: 215 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 215 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean)
+ Reducer 3
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: tinyint), VALUE._col1 (type: smallint), VALUE._col2 (type: int), VALUE._col3 (type: bigint), VALUE._col4 (type: float), VALUE._col5 (type: double), VALUE._col6 (type: string), VALUE._col7 (type: string), VALUE._col8 (type: timestamp), VALUE._col9 (type: timestamp), VALUE._col10 (type: boolean), VALUE._col11 (type: boolean)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11
+ Statistics: Num rows: 1 Data size: 215 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 1
+ Statistics: Num rows: 1 Data size: 215 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Right Outer Join0 to 1
+ filter predicates:
+ 0
+ 1 {(_col2 < 100)}
+ keys:
+ 0 UDFToInteger(_col0) (type: int)
+ 1 _col2 (type: int)
+ outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12
+ input vertices:
+ 0 Map 1
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
+ Select Operator
+ expressions: _col1 (type: tinyint), _col2 (type: smallint), _col3 (type: int), _col4 (type: bigint), _col5 (type: float), _col6 (type: double), _col7 (type: string), _col8 (type: string), _col9 (type: timestamp), _col10 (type: timestamp), _col11 (type: boolean), _col12 (type: boolean)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 1
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 1
+ Processor Tree:
+ ListSink
+
[10/12] hive git commit: HIVE-12450: OrcFileMergeOperator does not
use correct compression buffer size (Prasanth Jayachandran reviewed by Sergey
Shelukhin)
Posted by om...@apache.org.
HIVE-12450: OrcFileMergeOperator does not use correct compression buffer size (Prasanth Jayachandran reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2d25cf23
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2d25cf23
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2d25cf23
Branch: refs/heads/master-fixed
Commit: 2d25cf23abe535ba0ff48725f9e3d42479ffef4f
Parents: b9650be
Author: Prasanth Jayachandran <j....@gmail.com>
Authored: Fri Nov 20 01:24:25 2015 -0600
Committer: Owen O'Malley <om...@apache.org>
Committed: Fri Nov 20 13:46:23 2015 -0800
----------------------------------------------------------------------
.../test/resources/testconfiguration.properties | 2 +
.../hive/ql/exec/OrcFileMergeOperator.java | 20 +-
.../hive/ql/io/orc/OrcFileKeyWrapper.java | 6 +-
.../test/queries/clientpositive/orc_merge10.q | 110 +++
.../test/queries/clientpositive/orc_merge11.q | 43 ++
.../results/clientpositive/orc_merge10.q.out | 644 +++++++++++++++++
.../results/clientpositive/orc_merge11.q.out | 316 +++++++++
.../clientpositive/tez/orc_merge10.q.out | 709 +++++++++++++++++++
.../clientpositive/tez/orc_merge11.q.out | 316 +++++++++
9 files changed, 2155 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/2d25cf23/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index a33e720..b86dd19 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -155,6 +155,8 @@ minitez.query.files.shared=alter_merge_2_orc.q,\
orc_merge7.q,\
orc_merge8.q,\
orc_merge9.q,\
+ orc_merge10.q,\
+ orc_merge11.q,\
orc_merge_incompat1.q,\
orc_merge_incompat2.q,\
orc_vectorization_ppd.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/2d25cf23/ql/src/java/org/apache/hadoop/hive/ql/exec/OrcFileMergeOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/OrcFileMergeOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/OrcFileMergeOperator.java
index 2ea6154..99a3e8d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/OrcFileMergeOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/OrcFileMergeOperator.java
@@ -46,7 +46,7 @@ public class OrcFileMergeOperator extends
// does not merge, the file will be put into incompatible file set and will
// not be merged.
CompressionKind compression = null;
- long compressBuffSize = 0;
+ int compressBuffSize = 0;
OrcFile.Version version;
int columnCount = 0;
int rowIndexStride = 0;
@@ -104,13 +104,17 @@ public class OrcFileMergeOperator extends
columnCount = k.getTypes().get(0).getSubtypesCount();
rowIndexStride = k.getRowIndexStride();
- // block size and stripe size will be from config
- outWriter = OrcFile.createWriter(outPath,
- OrcFile.writerOptions(jc)
- .compress(compression)
- .version(version)
- .rowIndexStride(rowIndexStride)
- .inspector(reader.getObjectInspector()));
+ OrcFile.WriterOptions options = OrcFile.writerOptions(jc)
+ .compress(compression)
+ .version(version)
+ .rowIndexStride(rowIndexStride)
+ .inspector(reader.getObjectInspector());
+ // compression buffer size should only be set if compression is enabled
+ if (compression != CompressionKind.NONE) {
+ options.bufferSize(compressBuffSize);
+ }
+
+ outWriter = OrcFile.createWriter(outPath, options);
if (isLogDebugEnabled) {
LOG.info("ORC merge file output path: " + outPath);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/2d25cf23/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileKeyWrapper.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileKeyWrapper.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileKeyWrapper.java
index a62fc1e..8eda37f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileKeyWrapper.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileKeyWrapper.java
@@ -34,7 +34,7 @@ public class OrcFileKeyWrapper implements WritableComparable<OrcFileKeyWrapper>
private Path inputPath;
private CompressionKind compression;
- private long compressBufferSize;
+ private int compressBufferSize;
private List<OrcProto.Type> types;
private int rowIndexStride;
private OrcFile.Version version;
@@ -64,11 +64,11 @@ public class OrcFileKeyWrapper implements WritableComparable<OrcFileKeyWrapper>
this.rowIndexStride = rowIndexStride;
}
- public long getCompressBufferSize() {
+ public int getCompressBufferSize() {
return compressBufferSize;
}
- public void setCompressBufferSize(long compressBufferSize) {
+ public void setCompressBufferSize(int compressBufferSize) {
this.compressBufferSize = compressBufferSize;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/2d25cf23/ql/src/test/queries/clientpositive/orc_merge10.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/orc_merge10.q b/ql/src/test/queries/clientpositive/orc_merge10.q
new file mode 100644
index 0000000..7f81947
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/orc_merge10.q
@@ -0,0 +1,110 @@
+set hive.explain.user=false;
+set hive.merge.orcfile.stripe.level=false;
+set hive.exec.dynamic.partition=true;
+set hive.exec.dynamic.partition.mode=nonstrict;
+set hive.optimize.sort.dynamic.partition=false;
+set mapred.min.split.size=1000;
+set mapred.max.split.size=2000;
+set tez.am.grouping.split-count=2;
+set tez.grouping.split-count=2;
+set hive.merge.tezfiles=false;
+set hive.merge.mapfiles=false;
+set hive.merge.mapredfiles=false;
+
+-- SORT_QUERY_RESULTS
+
+DROP TABLE orcfile_merge1;
+DROP TABLE orcfile_merge1b;
+DROP TABLE orcfile_merge1c;
+
+CREATE TABLE orcfile_merge1 (key INT, value STRING)
+ PARTITIONED BY (ds STRING, part STRING) STORED AS ORC tblproperties("orc.compress"="SNAPPY","orc.compress.size"="4096");
+CREATE TABLE orcfile_merge1b (key INT, value STRING)
+ PARTITIONED BY (ds STRING, part STRING) STORED AS ORC tblproperties("orc.compress"="SNAPPY","orc.compress.size"="4096");
+CREATE TABLE orcfile_merge1c (key INT, value STRING)
+ PARTITIONED BY (ds STRING, part STRING) STORED AS ORC tblproperties("orc.compress"="SNAPPY","orc.compress.size"="4096");
+
+-- merge disabled
+EXPLAIN
+ INSERT OVERWRITE TABLE orcfile_merge1 PARTITION (ds='1', part)
+ SELECT key, value, PMOD(HASH(key), 2) as part
+ FROM src;
+
+INSERT OVERWRITE TABLE orcfile_merge1 PARTITION (ds='1', part)
+ SELECT key, value, PMOD(HASH(key), 2) as part
+ FROM src;
+
+dfs -ls ${hiveconf:hive.metastore.warehouse.dir}/orcfile_merge1/ds=1/part=0/;
+
+set hive.merge.tezfiles=true;
+set hive.merge.mapfiles=true;
+set hive.merge.mapredfiles=true;
+-- auto-merge slow way
+EXPLAIN
+ INSERT OVERWRITE TABLE orcfile_merge1b PARTITION (ds='1', part)
+ SELECT key, value, PMOD(HASH(key), 2) as part
+ FROM src;
+
+INSERT OVERWRITE TABLE orcfile_merge1b PARTITION (ds='1', part)
+ SELECT key, value, PMOD(HASH(key), 2) as part
+ FROM src;
+
+dfs -ls ${hiveconf:hive.metastore.warehouse.dir}/orcfile_merge1b/ds=1/part=0/;
+
+set hive.merge.orcfile.stripe.level=true;
+-- auto-merge fast way
+EXPLAIN
+ INSERT OVERWRITE TABLE orcfile_merge1c PARTITION (ds='1', part)
+ SELECT key, value, PMOD(HASH(key), 2) as part
+ FROM src;
+
+INSERT OVERWRITE TABLE orcfile_merge1c PARTITION (ds='1', part)
+ SELECT key, value, PMOD(HASH(key), 2) as part
+ FROM src;
+
+dfs -ls ${hiveconf:hive.metastore.warehouse.dir}/orcfile_merge1c/ds=1/part=0/;
+
+set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
+-- Verify
+SELECT SUM(HASH(c)) FROM (
+ SELECT TRANSFORM(*) USING 'tr \t _' AS (c)
+ FROM orcfile_merge1 WHERE ds='1'
+) t;
+
+SELECT SUM(HASH(c)) FROM (
+ SELECT TRANSFORM(*) USING 'tr \t _' AS (c)
+ FROM orcfile_merge1b WHERE ds='1'
+) t;
+
+select count(*) from orcfile_merge1;
+select count(*) from orcfile_merge1b;
+
+set tez.am.grouping.split-count=1;
+set tez.grouping.split-count=1;
+-- concatenate
+explain ALTER TABLE orcfile_merge1 PARTITION (ds='1', part='0') CONCATENATE;
+ALTER TABLE orcfile_merge1 PARTITION (ds='1', part='0') CONCATENATE;
+
+dfs -ls ${hiveconf:hive.metastore.warehouse.dir}/orcfile_merge1/ds=1/part=0/;
+
+-- Verify
+SELECT SUM(HASH(c)) FROM (
+ SELECT TRANSFORM(*) USING 'tr \t _' AS (c)
+ FROM orcfile_merge1c WHERE ds='1'
+) t;
+
+SELECT SUM(HASH(c)) FROM (
+ SELECT TRANSFORM(*) USING 'tr \t _' AS (c)
+ FROM orcfile_merge1 WHERE ds='1'
+) t;
+
+select count(*) from orcfile_merge1;
+select count(*) from orcfile_merge1c;
+
+SET hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.PostExecOrcFileDump;
+select * from orcfile_merge1 where ds='1' and part='0' limit 1;
+select * from orcfile_merge1c where ds='1' and part='0' limit 1;
+
+DROP TABLE orcfile_merge1;
+DROP TABLE orcfile_merge1b;
+DROP TABLE orcfile_merge1c;
http://git-wip-us.apache.org/repos/asf/hive/blob/2d25cf23/ql/src/test/queries/clientpositive/orc_merge11.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/orc_merge11.q b/ql/src/test/queries/clientpositive/orc_merge11.q
new file mode 100644
index 0000000..91f1991
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/orc_merge11.q
@@ -0,0 +1,43 @@
+DROP TABLE orcfile_merge1;
+DROP TABLE orc_split_elim;
+
+create table orc_split_elim (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) stored as orc;
+
+load data local inpath '../../data/files/orc_split_elim.orc' into table orc_split_elim;
+load data local inpath '../../data/files/orc_split_elim.orc' into table orc_split_elim;
+
+create table orcfile_merge1 (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) stored as orc tblproperties("orc.compress.size"="4096");
+
+insert overwrite table orcfile_merge1 select * from orc_split_elim;
+insert into table orcfile_merge1 select * from orc_split_elim;
+
+dfs -ls ${hiveconf:hive.metastore.warehouse.dir}/orcfile_merge1/;
+
+set hive.merge.tezfiles=true;
+set hive.merge.mapfiles=true;
+set hive.merge.mapredfiles=true;
+set hive.merge.orcfile.stripe.level=true;
+set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
+set tez.am.grouping.split-count=1;
+set tez.grouping.split-count=1;
+set hive.exec.orc.default.buffer.size=120;
+
+SET hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.PostExecOrcFileDump;
+select * from orcfile_merge1 limit 1;
+SET hive.exec.post.hooks=;
+
+-- concatenate
+ALTER TABLE orcfile_merge1 CONCATENATE;
+
+dfs -ls ${hiveconf:hive.metastore.warehouse.dir}/orcfile_merge1/;
+
+select count(*) from orc_split_elim;
+-- will have double the number of rows
+select count(*) from orcfile_merge1;
+
+SET hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.PostExecOrcFileDump;
+select * from orcfile_merge1 limit 1;
+SET hive.exec.post.hooks=;
+
+DROP TABLE orc_split_elim;
+DROP TABLE orcfile_merge1;
http://git-wip-us.apache.org/repos/asf/hive/blob/2d25cf23/ql/src/test/results/clientpositive/orc_merge10.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/orc_merge10.q.out b/ql/src/test/results/clientpositive/orc_merge10.q.out
new file mode 100644
index 0000000..89d0bf6
--- /dev/null
+++ b/ql/src/test/results/clientpositive/orc_merge10.q.out
@@ -0,0 +1,644 @@
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+DROP TABLE orcfile_merge1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+DROP TABLE orcfile_merge1
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE orcfile_merge1b
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE orcfile_merge1b
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE orcfile_merge1c
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE orcfile_merge1c
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE orcfile_merge1 (key INT, value STRING)
+ PARTITIONED BY (ds STRING, part STRING) STORED AS ORC tblproperties("orc.compress"="SNAPPY","orc.compress.size"="4096")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@orcfile_merge1
+POSTHOOK: query: CREATE TABLE orcfile_merge1 (key INT, value STRING)
+ PARTITIONED BY (ds STRING, part STRING) STORED AS ORC tblproperties("orc.compress"="SNAPPY","orc.compress.size"="4096")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@orcfile_merge1
+PREHOOK: query: CREATE TABLE orcfile_merge1b (key INT, value STRING)
+ PARTITIONED BY (ds STRING, part STRING) STORED AS ORC tblproperties("orc.compress"="SNAPPY","orc.compress.size"="4096")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@orcfile_merge1b
+POSTHOOK: query: CREATE TABLE orcfile_merge1b (key INT, value STRING)
+ PARTITIONED BY (ds STRING, part STRING) STORED AS ORC tblproperties("orc.compress"="SNAPPY","orc.compress.size"="4096")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@orcfile_merge1b
+PREHOOK: query: CREATE TABLE orcfile_merge1c (key INT, value STRING)
+ PARTITIONED BY (ds STRING, part STRING) STORED AS ORC tblproperties("orc.compress"="SNAPPY","orc.compress.size"="4096")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@orcfile_merge1c
+POSTHOOK: query: CREATE TABLE orcfile_merge1c (key INT, value STRING)
+ PARTITIONED BY (ds STRING, part STRING) STORED AS ORC tblproperties("orc.compress"="SNAPPY","orc.compress.size"="4096")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@orcfile_merge1c
+PREHOOK: query: -- merge disabled
+EXPLAIN
+ INSERT OVERWRITE TABLE orcfile_merge1 PARTITION (ds='1', part)
+ SELECT key, value, PMOD(HASH(key), 2) as part
+ FROM src
+PREHOOK: type: QUERY
+POSTHOOK: query: -- merge disabled
+EXPLAIN
+ INSERT OVERWRITE TABLE orcfile_merge1 PARTITION (ds='1', part)
+ SELECT key, value, PMOD(HASH(key), 2) as part
+ FROM src
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+ Stage-2 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.orcfile_merge1
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ partition:
+ ds 1
+ part
+ replace: true
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.orcfile_merge1
+
+ Stage: Stage-2
+ Stats-Aggr Operator
+
+PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge1 PARTITION (ds='1', part)
+ SELECT key, value, PMOD(HASH(key), 2) as part
+ FROM src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@orcfile_merge1@ds=1
+POSTHOOK: query: INSERT OVERWRITE TABLE orcfile_merge1 PARTITION (ds='1', part)
+ SELECT key, value, PMOD(HASH(key), 2) as part
+ FROM src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@orcfile_merge1@ds=1/part=0
+POSTHOOK: Output: default@orcfile_merge1@ds=1/part=1
+POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=0).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=0).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+Found 2 items
+#### A masked pattern was here ####
+PREHOOK: query: -- auto-merge slow way
+EXPLAIN
+ INSERT OVERWRITE TABLE orcfile_merge1b PARTITION (ds='1', part)
+ SELECT key, value, PMOD(HASH(key), 2) as part
+ FROM src
+PREHOOK: type: QUERY
+POSTHOOK: query: -- auto-merge slow way
+EXPLAIN
+ INSERT OVERWRITE TABLE orcfile_merge1b PARTITION (ds='1', part)
+ SELECT key, value, PMOD(HASH(key), 2) as part
+ FROM src
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5
+ Stage-4
+ Stage-0 depends on stages: Stage-4, Stage-3, Stage-6
+ Stage-2 depends on stages: Stage-0
+ Stage-3
+ Stage-5
+ Stage-6 depends on stages: Stage-5
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.orcfile_merge1b
+
+ Stage: Stage-7
+ Conditional Operator
+
+ Stage: Stage-4
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ partition:
+ ds 1
+ part
+ replace: true
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.orcfile_merge1b
+
+ Stage: Stage-2
+ Stats-Aggr Operator
+
+ Stage: Stage-3
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.orcfile_merge1b
+
+ Stage: Stage-5
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.orcfile_merge1b
+
+ Stage: Stage-6
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
+
+PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge1b PARTITION (ds='1', part)
+ SELECT key, value, PMOD(HASH(key), 2) as part
+ FROM src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@orcfile_merge1b@ds=1
+POSTHOOK: query: INSERT OVERWRITE TABLE orcfile_merge1b PARTITION (ds='1', part)
+ SELECT key, value, PMOD(HASH(key), 2) as part
+ FROM src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=0
+POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=1
+POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=0).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=0).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+Found 1 items
+#### A masked pattern was here ####
+PREHOOK: query: -- auto-merge fast way
+EXPLAIN
+ INSERT OVERWRITE TABLE orcfile_merge1c PARTITION (ds='1', part)
+ SELECT key, value, PMOD(HASH(key), 2) as part
+ FROM src
+PREHOOK: type: QUERY
+POSTHOOK: query: -- auto-merge fast way
+EXPLAIN
+ INSERT OVERWRITE TABLE orcfile_merge1c PARTITION (ds='1', part)
+ SELECT key, value, PMOD(HASH(key), 2) as part
+ FROM src
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5
+ Stage-4
+ Stage-0 depends on stages: Stage-4, Stage-3, Stage-6
+ Stage-2 depends on stages: Stage-0
+ Stage-3
+ Stage-5
+ Stage-6 depends on stages: Stage-5
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.orcfile_merge1c
+
+ Stage: Stage-7
+ Conditional Operator
+
+ Stage: Stage-4
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ partition:
+ ds 1
+ part
+ replace: true
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.orcfile_merge1c
+
+ Stage: Stage-2
+ Stats-Aggr Operator
+
+ Stage: Stage-3
+ Merge File Operator
+ Map Operator Tree:
+ ORC File Merge Operator
+ merge level: stripe
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+
+ Stage: Stage-5
+ Merge File Operator
+ Map Operator Tree:
+ ORC File Merge Operator
+ merge level: stripe
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+
+ Stage: Stage-6
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
+
+PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge1c PARTITION (ds='1', part)
+ SELECT key, value, PMOD(HASH(key), 2) as part
+ FROM src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@orcfile_merge1c@ds=1
+POSTHOOK: query: INSERT OVERWRITE TABLE orcfile_merge1c PARTITION (ds='1', part)
+ SELECT key, value, PMOD(HASH(key), 2) as part
+ FROM src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@orcfile_merge1c@ds=1/part=0
+POSTHOOK: Output: default@orcfile_merge1c@ds=1/part=1
+POSTHOOK: Lineage: orcfile_merge1c PARTITION(ds=1,part=0).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: orcfile_merge1c PARTITION(ds=1,part=0).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: orcfile_merge1c PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: orcfile_merge1c PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+Found 1 items
+#### A masked pattern was here ####
+PREHOOK: query: -- Verify
+SELECT SUM(HASH(c)) FROM (
+ SELECT TRANSFORM(*) USING 'tr \t _' AS (c)
+ FROM orcfile_merge1 WHERE ds='1'
+) t
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orcfile_merge1
+PREHOOK: Input: default@orcfile_merge1@ds=1/part=0
+PREHOOK: Input: default@orcfile_merge1@ds=1/part=1
+#### A masked pattern was here ####
+POSTHOOK: query: -- Verify
+SELECT SUM(HASH(c)) FROM (
+ SELECT TRANSFORM(*) USING 'tr \t _' AS (c)
+ FROM orcfile_merge1 WHERE ds='1'
+) t
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orcfile_merge1
+POSTHOOK: Input: default@orcfile_merge1@ds=1/part=0
+POSTHOOK: Input: default@orcfile_merge1@ds=1/part=1
+#### A masked pattern was here ####
+-21975308766
+PREHOOK: query: SELECT SUM(HASH(c)) FROM (
+ SELECT TRANSFORM(*) USING 'tr \t _' AS (c)
+ FROM orcfile_merge1b WHERE ds='1'
+) t
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orcfile_merge1b
+PREHOOK: Input: default@orcfile_merge1b@ds=1/part=0
+PREHOOK: Input: default@orcfile_merge1b@ds=1/part=1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT SUM(HASH(c)) FROM (
+ SELECT TRANSFORM(*) USING 'tr \t _' AS (c)
+ FROM orcfile_merge1b WHERE ds='1'
+) t
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orcfile_merge1b
+POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=0
+POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=1
+#### A masked pattern was here ####
+-21975308766
+PREHOOK: query: select count(*) from orcfile_merge1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orcfile_merge1
+PREHOOK: Input: default@orcfile_merge1@ds=1/part=0
+PREHOOK: Input: default@orcfile_merge1@ds=1/part=1
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from orcfile_merge1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orcfile_merge1
+POSTHOOK: Input: default@orcfile_merge1@ds=1/part=0
+POSTHOOK: Input: default@orcfile_merge1@ds=1/part=1
+#### A masked pattern was here ####
+500
+PREHOOK: query: select count(*) from orcfile_merge1b
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orcfile_merge1b
+PREHOOK: Input: default@orcfile_merge1b@ds=1/part=0
+PREHOOK: Input: default@orcfile_merge1b@ds=1/part=1
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from orcfile_merge1b
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orcfile_merge1b
+POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=0
+POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=1
+#### A masked pattern was here ####
+500
+PREHOOK: query: -- concatenate
+explain ALTER TABLE orcfile_merge1 PARTITION (ds='1', part='0') CONCATENATE
+PREHOOK: type: ALTER_PARTITION_MERGE
+POSTHOOK: query: -- concatenate
+explain ALTER TABLE orcfile_merge1 PARTITION (ds='1', part='0') CONCATENATE
+POSTHOOK: type: ALTER_PARTITION_MERGE
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+ Stage-1 depends on stages: Stage-0
+ Stage-2 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-0
+
+ Stage: Stage-1
+ Move Operator
+ tables:
+ partition:
+ ds 1
+ part 0
+ replace: true
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.orcfile_merge1
+
+ Stage: Stage-2
+ Stats-Aggr Operator
+
+PREHOOK: query: ALTER TABLE orcfile_merge1 PARTITION (ds='1', part='0') CONCATENATE
+PREHOOK: type: ALTER_PARTITION_MERGE
+PREHOOK: Input: default@orcfile_merge1
+PREHOOK: Output: default@orcfile_merge1@ds=1/part=0
+POSTHOOK: query: ALTER TABLE orcfile_merge1 PARTITION (ds='1', part='0') CONCATENATE
+POSTHOOK: type: ALTER_PARTITION_MERGE
+POSTHOOK: Input: default@orcfile_merge1
+POSTHOOK: Output: default@orcfile_merge1@ds=1/part=0
+Found 1 items
+#### A masked pattern was here ####
+PREHOOK: query: -- Verify
+SELECT SUM(HASH(c)) FROM (
+ SELECT TRANSFORM(*) USING 'tr \t _' AS (c)
+ FROM orcfile_merge1c WHERE ds='1'
+) t
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orcfile_merge1c
+PREHOOK: Input: default@orcfile_merge1c@ds=1/part=0
+PREHOOK: Input: default@orcfile_merge1c@ds=1/part=1
+#### A masked pattern was here ####
+POSTHOOK: query: -- Verify
+SELECT SUM(HASH(c)) FROM (
+ SELECT TRANSFORM(*) USING 'tr \t _' AS (c)
+ FROM orcfile_merge1c WHERE ds='1'
+) t
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orcfile_merge1c
+POSTHOOK: Input: default@orcfile_merge1c@ds=1/part=0
+POSTHOOK: Input: default@orcfile_merge1c@ds=1/part=1
+#### A masked pattern was here ####
+-21975308766
+PREHOOK: query: SELECT SUM(HASH(c)) FROM (
+ SELECT TRANSFORM(*) USING 'tr \t _' AS (c)
+ FROM orcfile_merge1 WHERE ds='1'
+) t
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orcfile_merge1
+PREHOOK: Input: default@orcfile_merge1@ds=1/part=0
+PREHOOK: Input: default@orcfile_merge1@ds=1/part=1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT SUM(HASH(c)) FROM (
+ SELECT TRANSFORM(*) USING 'tr \t _' AS (c)
+ FROM orcfile_merge1 WHERE ds='1'
+) t
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orcfile_merge1
+POSTHOOK: Input: default@orcfile_merge1@ds=1/part=0
+POSTHOOK: Input: default@orcfile_merge1@ds=1/part=1
+#### A masked pattern was here ####
+-21975308766
+PREHOOK: query: select count(*) from orcfile_merge1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orcfile_merge1
+PREHOOK: Input: default@orcfile_merge1@ds=1/part=0
+PREHOOK: Input: default@orcfile_merge1@ds=1/part=1
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from orcfile_merge1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orcfile_merge1
+POSTHOOK: Input: default@orcfile_merge1@ds=1/part=0
+POSTHOOK: Input: default@orcfile_merge1@ds=1/part=1
+#### A masked pattern was here ####
+500
+PREHOOK: query: select count(*) from orcfile_merge1c
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orcfile_merge1c
+PREHOOK: Input: default@orcfile_merge1c@ds=1/part=0
+PREHOOK: Input: default@orcfile_merge1c@ds=1/part=1
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from orcfile_merge1c
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orcfile_merge1c
+POSTHOOK: Input: default@orcfile_merge1c@ds=1/part=0
+POSTHOOK: Input: default@orcfile_merge1c@ds=1/part=1
+#### A masked pattern was here ####
+500
+PREHOOK: query: select * from orcfile_merge1 where ds='1' and part='0' limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orcfile_merge1
+PREHOOK: Input: default@orcfile_merge1@ds=1/part=0
+#### A masked pattern was here ####
+-- BEGIN ORC FILE DUMP --
+#### A masked pattern was here ####
+File Version: 0.12 with HIVE_4243
+Rows: 242
+Compression: SNAPPY
+Compression size: 4096
+Type: struct<key:int,value:string>
+
+Stripe Statistics:
+ Stripe 1:
+ Column 0: count: 152 hasNull: false
+ Column 1: count: 152 hasNull: false min: 0 max: 497 sum: 38034
+ Column 2: count: 152 hasNull: false min: val_0 max: val_97 sum: 1034
+ Stripe 2:
+ Column 0: count: 90 hasNull: false
+ Column 1: count: 90 hasNull: false min: 0 max: 495 sum: 22736
+ Column 2: count: 90 hasNull: false min: val_0 max: val_86 sum: 612
+
+File Statistics:
+ Column 0: count: 242 hasNull: false
+ Column 1: count: 242 hasNull: false min: 0 max: 497 sum: 60770
+ Column 2: count: 242 hasNull: false min: val_0 max: val_97 sum: 1646
+
+Stripes:
+ Stripe: offset: 3 data: 988 rows: 152 tail: 72 index: 77
+ Stream: column 0 section ROW_INDEX start: 3 length 12
+ Stream: column 1 section ROW_INDEX start: 15 length 28
+ Stream: column 2 section ROW_INDEX start: 43 length 37
+ Stream: column 1 section DATA start: 80 length 309
+ Stream: column 2 section DATA start: 389 length 157
+ Stream: column 2 section LENGTH start: 546 length 60
+ Stream: column 2 section DICTIONARY_DATA start: 606 length 462
+ Encoding column 0: DIRECT
+ Encoding column 1: DIRECT_V2
+ Encoding column 2: DICTIONARY_V2[114]
+ Row group indices for column 1:
+ Entry 0: count: 152 hasNull: false min: 0 max: 497 sum: 38034 positions: 0,0,0
+ Stripe: offset: 1140 data: 616 rows: 90 tail: 61 index: 76
+ Stream: column 0 section ROW_INDEX start: 1140 length 11
+ Stream: column 1 section ROW_INDEX start: 1151 length 27
+ Stream: column 2 section ROW_INDEX start: 1178 length 38
+ Stream: column 1 section DATA start: 1216 length 185
+ Stream: column 2 section DATA start: 1401 length 377
+ Stream: column 2 section LENGTH start: 1778 length 54
+ Encoding column 0: DIRECT
+ Encoding column 1: DIRECT_V2
+ Encoding column 2: DIRECT_V2
+ Row group indices for column 1:
+ Entry 0: count: 90 hasNull: false min: 0 max: 495 sum: 22736 positions: 0,0,0
+
+File length: 2137 bytes
+Padding length: 0 bytes
+Padding ratio: 0%
+-- END ORC FILE DUMP --
+172 val_172 1 0
+PREHOOK: query: select * from orcfile_merge1c where ds='1' and part='0' limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orcfile_merge1c
+PREHOOK: Input: default@orcfile_merge1c@ds=1/part=0
+#### A masked pattern was here ####
+-- BEGIN ORC FILE DUMP --
+#### A masked pattern was here ####
+File Version: 0.12 with HIVE_4243
+Rows: 242
+Compression: SNAPPY
+Compression size: 4096
+Type: struct<key:int,value:string>
+
+Stripe Statistics:
+ Stripe 1:
+ Column 0: count: 152 hasNull: false
+ Column 1: count: 152 hasNull: false min: 0 max: 497 sum: 38034
+ Column 2: count: 152 hasNull: false min: val_0 max: val_97 sum: 1034
+ Stripe 2:
+ Column 0: count: 90 hasNull: false
+ Column 1: count: 90 hasNull: false min: 0 max: 495 sum: 22736
+ Column 2: count: 90 hasNull: false min: val_0 max: val_86 sum: 612
+
+File Statistics:
+ Column 0: count: 242 hasNull: false
+ Column 1: count: 242 hasNull: false min: 0 max: 497 sum: 60770
+ Column 2: count: 242 hasNull: false min: val_0 max: val_97 sum: 1646
+
+Stripes:
+ Stripe: offset: 3 data: 988 rows: 152 tail: 72 index: 77
+ Stream: column 0 section ROW_INDEX start: 3 length 12
+ Stream: column 1 section ROW_INDEX start: 15 length 28
+ Stream: column 2 section ROW_INDEX start: 43 length 37
+ Stream: column 1 section DATA start: 80 length 309
+ Stream: column 2 section DATA start: 389 length 157
+ Stream: column 2 section LENGTH start: 546 length 60
+ Stream: column 2 section DICTIONARY_DATA start: 606 length 462
+ Encoding column 0: DIRECT
+ Encoding column 1: DIRECT_V2
+ Encoding column 2: DICTIONARY_V2[114]
+ Row group indices for column 1:
+ Entry 0: count: 152 hasNull: false min: 0 max: 497 sum: 38034 positions: 0,0,0
+ Stripe: offset: 1140 data: 616 rows: 90 tail: 61 index: 76
+ Stream: column 0 section ROW_INDEX start: 1140 length 11
+ Stream: column 1 section ROW_INDEX start: 1151 length 27
+ Stream: column 2 section ROW_INDEX start: 1178 length 38
+ Stream: column 1 section DATA start: 1216 length 185
+ Stream: column 2 section DATA start: 1401 length 377
+ Stream: column 2 section LENGTH start: 1778 length 54
+ Encoding column 0: DIRECT
+ Encoding column 1: DIRECT_V2
+ Encoding column 2: DIRECT_V2
+ Row group indices for column 1:
+ Entry 0: count: 90 hasNull: false min: 0 max: 495 sum: 22736 positions: 0,0,0
+
+File length: 2137 bytes
+Padding length: 0 bytes
+Padding ratio: 0%
+-- END ORC FILE DUMP --
+172 val_172 1 0
+PREHOOK: query: DROP TABLE orcfile_merge1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@orcfile_merge1
+PREHOOK: Output: default@orcfile_merge1
+PREHOOK: query: DROP TABLE orcfile_merge1b
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@orcfile_merge1b
+PREHOOK: Output: default@orcfile_merge1b
+PREHOOK: query: DROP TABLE orcfile_merge1c
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@orcfile_merge1c
+PREHOOK: Output: default@orcfile_merge1c
http://git-wip-us.apache.org/repos/asf/hive/blob/2d25cf23/ql/src/test/results/clientpositive/orc_merge11.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/orc_merge11.q.out b/ql/src/test/results/clientpositive/orc_merge11.q.out
new file mode 100644
index 0000000..da608db
--- /dev/null
+++ b/ql/src/test/results/clientpositive/orc_merge11.q.out
@@ -0,0 +1,316 @@
+PREHOOK: query: DROP TABLE orcfile_merge1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE orcfile_merge1
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE orc_split_elim
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE orc_split_elim
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table orc_split_elim (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) stored as orc
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@orc_split_elim
+POSTHOOK: query: create table orc_split_elim (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) stored as orc
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@orc_split_elim
+PREHOOK: query: load data local inpath '../../data/files/orc_split_elim.orc' into table orc_split_elim
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@orc_split_elim
+POSTHOOK: query: load data local inpath '../../data/files/orc_split_elim.orc' into table orc_split_elim
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@orc_split_elim
+PREHOOK: query: load data local inpath '../../data/files/orc_split_elim.orc' into table orc_split_elim
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@orc_split_elim
+POSTHOOK: query: load data local inpath '../../data/files/orc_split_elim.orc' into table orc_split_elim
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@orc_split_elim
+PREHOOK: query: create table orcfile_merge1 (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) stored as orc tblproperties("orc.compress.size"="4096")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@orcfile_merge1
+POSTHOOK: query: create table orcfile_merge1 (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) stored as orc tblproperties("orc.compress.size"="4096")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@orcfile_merge1
+PREHOOK: query: insert overwrite table orcfile_merge1 select * from orc_split_elim
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_split_elim
+PREHOOK: Output: default@orcfile_merge1
+POSTHOOK: query: insert overwrite table orcfile_merge1 select * from orc_split_elim
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_split_elim
+POSTHOOK: Output: default@orcfile_merge1
+POSTHOOK: Lineage: orcfile_merge1.decimal1 SIMPLE [(orc_split_elim)orc_split_elim.FieldSchema(name:decimal1, type:decimal(10,0), comment:null), ]
+POSTHOOK: Lineage: orcfile_merge1.string1 SIMPLE [(orc_split_elim)orc_split_elim.FieldSchema(name:string1, type:string, comment:null), ]
+POSTHOOK: Lineage: orcfile_merge1.subtype SIMPLE [(orc_split_elim)orc_split_elim.FieldSchema(name:subtype, type:double, comment:null), ]
+POSTHOOK: Lineage: orcfile_merge1.ts SIMPLE [(orc_split_elim)orc_split_elim.FieldSchema(name:ts, type:timestamp, comment:null), ]
+POSTHOOK: Lineage: orcfile_merge1.userid SIMPLE [(orc_split_elim)orc_split_elim.FieldSchema(name:userid, type:bigint, comment:null), ]
+PREHOOK: query: insert into table orcfile_merge1 select * from orc_split_elim
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_split_elim
+PREHOOK: Output: default@orcfile_merge1
+POSTHOOK: query: insert into table orcfile_merge1 select * from orc_split_elim
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_split_elim
+POSTHOOK: Output: default@orcfile_merge1
+POSTHOOK: Lineage: orcfile_merge1.decimal1 SIMPLE [(orc_split_elim)orc_split_elim.FieldSchema(name:decimal1, type:decimal(10,0), comment:null), ]
+POSTHOOK: Lineage: orcfile_merge1.string1 SIMPLE [(orc_split_elim)orc_split_elim.FieldSchema(name:string1, type:string, comment:null), ]
+POSTHOOK: Lineage: orcfile_merge1.subtype SIMPLE [(orc_split_elim)orc_split_elim.FieldSchema(name:subtype, type:double, comment:null), ]
+POSTHOOK: Lineage: orcfile_merge1.ts SIMPLE [(orc_split_elim)orc_split_elim.FieldSchema(name:ts, type:timestamp, comment:null), ]
+POSTHOOK: Lineage: orcfile_merge1.userid SIMPLE [(orc_split_elim)orc_split_elim.FieldSchema(name:userid, type:bigint, comment:null), ]
+Found 2 items
+#### A masked pattern was here ####
+PREHOOK: query: select * from orcfile_merge1 limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orcfile_merge1
+#### A masked pattern was here ####
+-- BEGIN ORC FILE DUMP --
+#### A masked pattern was here ####
+File Version: 0.12 with HIVE_4243
+Rows: 50000
+Compression: ZLIB
+Compression size: 4096
+Type: struct<userid:bigint,string1:string,subtype:double,decimal1:decimal(10,0),ts:timestamp>
+
+Stripe Statistics:
+ Stripe 1:
+ Column 0: count: 50000 hasNull: false
+ Column 1: count: 50000 hasNull: false min: 2 max: 100 sum: 4999238
+ Column 2: count: 50000 hasNull: false min: bar max: zebra sum: 249980
+ Column 3: count: 50000 hasNull: false min: 0.8 max: 80.0 sum: 400102.80000000005
+ Column 4: count: 50000 hasNull: false min: 0 max: 6 sum: 32
+ Column 5: count: 50000 hasNull: false min: 1969-12-31 16:00:00.0 max: 1969-12-31 16:04:10.0
+
+File Statistics:
+ Column 0: count: 50000 hasNull: false
+ Column 1: count: 50000 hasNull: false min: 2 max: 100 sum: 4999238
+ Column 2: count: 50000 hasNull: false min: bar max: zebra sum: 249980
+ Column 3: count: 50000 hasNull: false min: 0.8 max: 80.0 sum: 400102.80000000005
+ Column 4: count: 50000 hasNull: false min: 0 max: 6 sum: 32
+ Column 5: count: 50000 hasNull: false min: 1969-12-31 16:00:00.0 max: 1969-12-31 16:04:10.0
+
+Stripes:
+ Stripe: offset: 3 data: 10104 rows: 50000 tail: 117 index: 509
+ Stream: column 0 section ROW_INDEX start: 3 length 17
+ Stream: column 1 section ROW_INDEX start: 20 length 85
+ Stream: column 2 section ROW_INDEX start: 105 length 87
+ Stream: column 3 section ROW_INDEX start: 192 length 111
+ Stream: column 4 section ROW_INDEX start: 303 length 108
+ Stream: column 5 section ROW_INDEX start: 411 length 101
+ Stream: column 1 section DATA start: 512 length 871
+ Stream: column 2 section DATA start: 1383 length 362
+ Stream: column 2 section LENGTH start: 1745 length 8
+ Stream: column 2 section DICTIONARY_DATA start: 1753 length 23
+ Stream: column 3 section DATA start: 1776 length 5167
+ Stream: column 4 section DATA start: 6943 length 524
+ Stream: column 4 section SECONDARY start: 7467 length 118
+ Stream: column 5 section DATA start: 7585 length 2913
+ Stream: column 5 section SECONDARY start: 10498 length 118
+ Encoding column 0: DIRECT
+ Encoding column 1: DIRECT_V2
+ Encoding column 2: DICTIONARY_V2[6]
+ Encoding column 3: DIRECT
+ Encoding column 4: DIRECT_V2
+ Encoding column 5: DIRECT_V2
+ Row group indices for column 1:
+ Entry 0: count: 10000 hasNull: false min: 2 max: 100 sum: 999815 positions: 0,0,0
+ Entry 1: count: 10000 hasNull: false min: 29 max: 100 sum: 999899 positions: 133,1071,391
+ Entry 2: count: 10000 hasNull: false min: 2 max: 100 sum: 999807 positions: 292,2147,391
+ Entry 3: count: 10000 hasNull: false min: 13 max: 100 sum: 999842 positions: 453,3223,391
+ Entry 4: count: 10000 hasNull: false min: 5 max: 100 sum: 999875 positions: 683,203,391
+
+File length: 11071 bytes
+Padding length: 0 bytes
+Padding ratio: 0%
+-- END ORC FILE DUMP --
+-- BEGIN ORC FILE DUMP --
+#### A masked pattern was here ####
+File Version: 0.12 with HIVE_4243
+Rows: 50000
+Compression: ZLIB
+Compression size: 4096
+Type: struct<userid:bigint,string1:string,subtype:double,decimal1:decimal(10,0),ts:timestamp>
+
+Stripe Statistics:
+ Stripe 1:
+ Column 0: count: 50000 hasNull: false
+ Column 1: count: 50000 hasNull: false min: 2 max: 100 sum: 4999238
+ Column 2: count: 50000 hasNull: false min: bar max: zebra sum: 249980
+ Column 3: count: 50000 hasNull: false min: 0.8 max: 80.0 sum: 400102.80000000005
+ Column 4: count: 50000 hasNull: false min: 0 max: 6 sum: 32
+ Column 5: count: 50000 hasNull: false min: 1969-12-31 16:00:00.0 max: 1969-12-31 16:04:10.0
+
+File Statistics:
+ Column 0: count: 50000 hasNull: false
+ Column 1: count: 50000 hasNull: false min: 2 max: 100 sum: 4999238
+ Column 2: count: 50000 hasNull: false min: bar max: zebra sum: 249980
+ Column 3: count: 50000 hasNull: false min: 0.8 max: 80.0 sum: 400102.80000000005
+ Column 4: count: 50000 hasNull: false min: 0 max: 6 sum: 32
+ Column 5: count: 50000 hasNull: false min: 1969-12-31 16:00:00.0 max: 1969-12-31 16:04:10.0
+
+Stripes:
+ Stripe: offset: 3 data: 10104 rows: 50000 tail: 117 index: 509
+ Stream: column 0 section ROW_INDEX start: 3 length 17
+ Stream: column 1 section ROW_INDEX start: 20 length 85
+ Stream: column 2 section ROW_INDEX start: 105 length 87
+ Stream: column 3 section ROW_INDEX start: 192 length 111
+ Stream: column 4 section ROW_INDEX start: 303 length 108
+ Stream: column 5 section ROW_INDEX start: 411 length 101
+ Stream: column 1 section DATA start: 512 length 871
+ Stream: column 2 section DATA start: 1383 length 362
+ Stream: column 2 section LENGTH start: 1745 length 8
+ Stream: column 2 section DICTIONARY_DATA start: 1753 length 23
+ Stream: column 3 section DATA start: 1776 length 5167
+ Stream: column 4 section DATA start: 6943 length 524
+ Stream: column 4 section SECONDARY start: 7467 length 118
+ Stream: column 5 section DATA start: 7585 length 2913
+ Stream: column 5 section SECONDARY start: 10498 length 118
+ Encoding column 0: DIRECT
+ Encoding column 1: DIRECT_V2
+ Encoding column 2: DICTIONARY_V2[6]
+ Encoding column 3: DIRECT
+ Encoding column 4: DIRECT_V2
+ Encoding column 5: DIRECT_V2
+ Row group indices for column 1:
+ Entry 0: count: 10000 hasNull: false min: 2 max: 100 sum: 999815 positions: 0,0,0
+ Entry 1: count: 10000 hasNull: false min: 29 max: 100 sum: 999899 positions: 133,1071,391
+ Entry 2: count: 10000 hasNull: false min: 2 max: 100 sum: 999807 positions: 292,2147,391
+ Entry 3: count: 10000 hasNull: false min: 13 max: 100 sum: 999842 positions: 453,3223,391
+ Entry 4: count: 10000 hasNull: false min: 5 max: 100 sum: 999875 positions: 683,203,391
+
+File length: 11071 bytes
+Padding length: 0 bytes
+Padding ratio: 0%
+-- END ORC FILE DUMP --
+2 foo 0.8 1 1969-12-31 16:00:00
+PREHOOK: query: -- concatenate
+ALTER TABLE orcfile_merge1 CONCATENATE
+PREHOOK: type: ALTER_TABLE_MERGE
+PREHOOK: Input: default@orcfile_merge1
+PREHOOK: Output: default@orcfile_merge1
+Found 1 items
+#### A masked pattern was here ####
+PREHOOK: query: select count(*) from orc_split_elim
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_split_elim
+#### A masked pattern was here ####
+50000
+PREHOOK: query: -- will have double the number of rows
+select count(*) from orcfile_merge1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orcfile_merge1
+#### A masked pattern was here ####
+100000
+PREHOOK: query: select * from orcfile_merge1 limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orcfile_merge1
+#### A masked pattern was here ####
+-- BEGIN ORC FILE DUMP --
+#### A masked pattern was here ####
+File Version: 0.12 with HIVE_4243
+Rows: 100000
+Compression: ZLIB
+Compression size: 4096
+Type: struct<userid:bigint,string1:string,subtype:double,decimal1:decimal(10,0),ts:timestamp>
+
+Stripe Statistics:
+ Stripe 1:
+ Column 0: count: 50000 hasNull: false
+ Column 1: count: 50000 hasNull: false min: 2 max: 100 sum: 4999238
+ Column 2: count: 50000 hasNull: false min: bar max: zebra sum: 249980
+ Column 3: count: 50000 hasNull: false min: 0.8 max: 80.0 sum: 400102.80000000005
+ Column 4: count: 50000 hasNull: false min: 0 max: 6 sum: 32
+ Column 5: count: 50000 hasNull: false min: 1969-12-31 16:00:00.0 max: 1969-12-31 16:04:10.0
+ Stripe 2:
+ Column 0: count: 50000 hasNull: false
+ Column 1: count: 50000 hasNull: false min: 2 max: 100 sum: 4999238
+ Column 2: count: 50000 hasNull: false min: bar max: zebra sum: 249980
+ Column 3: count: 50000 hasNull: false min: 0.8 max: 80.0 sum: 400102.80000000005
+ Column 4: count: 50000 hasNull: false min: 0 max: 6 sum: 32
+ Column 5: count: 50000 hasNull: false min: 1969-12-31 16:00:00.0 max: 1969-12-31 16:04:10.0
+
+File Statistics:
+ Column 0: count: 100000 hasNull: false
+ Column 1: count: 100000 hasNull: false min: 2 max: 100 sum: 9998476
+ Column 2: count: 100000 hasNull: false min: bar max: zebra sum: 499960
+ Column 3: count: 100000 hasNull: false min: 0.8 max: 80.0 sum: 800205.6000000001
+ Column 4: count: 100000 hasNull: false min: 0 max: 6 sum: 64
+ Column 5: count: 100000 hasNull: false min: 1969-12-31 16:00:00.0 max: 1969-12-31 16:04:10.0
+
+Stripes:
+ Stripe: offset: 3 data: 10104 rows: 50000 tail: 117 index: 509
+ Stream: column 0 section ROW_INDEX start: 3 length 17
+ Stream: column 1 section ROW_INDEX start: 20 length 85
+ Stream: column 2 section ROW_INDEX start: 105 length 87
+ Stream: column 3 section ROW_INDEX start: 192 length 111
+ Stream: column 4 section ROW_INDEX start: 303 length 108
+ Stream: column 5 section ROW_INDEX start: 411 length 101
+ Stream: column 1 section DATA start: 512 length 871
+ Stream: column 2 section DATA start: 1383 length 362
+ Stream: column 2 section LENGTH start: 1745 length 8
+ Stream: column 2 section DICTIONARY_DATA start: 1753 length 23
+ Stream: column 3 section DATA start: 1776 length 5167
+ Stream: column 4 section DATA start: 6943 length 524
+ Stream: column 4 section SECONDARY start: 7467 length 118
+ Stream: column 5 section DATA start: 7585 length 2913
+ Stream: column 5 section SECONDARY start: 10498 length 118
+ Encoding column 0: DIRECT
+ Encoding column 1: DIRECT_V2
+ Encoding column 2: DICTIONARY_V2[6]
+ Encoding column 3: DIRECT
+ Encoding column 4: DIRECT_V2
+ Encoding column 5: DIRECT_V2
+ Row group indices for column 1:
+ Entry 0: count: 10000 hasNull: false min: 2 max: 100 sum: 999815 positions: 0,0,0
+ Entry 1: count: 10000 hasNull: false min: 29 max: 100 sum: 999899 positions: 133,1071,391
+ Entry 2: count: 10000 hasNull: false min: 2 max: 100 sum: 999807 positions: 292,2147,391
+ Entry 3: count: 10000 hasNull: false min: 13 max: 100 sum: 999842 positions: 453,3223,391
+ Entry 4: count: 10000 hasNull: false min: 5 max: 100 sum: 999875 positions: 683,203,391
+ Stripe: offset: 10733 data: 10104 rows: 50000 tail: 117 index: 509
+ Stream: column 0 section ROW_INDEX start: 10733 length 17
+ Stream: column 1 section ROW_INDEX start: 10750 length 85
+ Stream: column 2 section ROW_INDEX start: 10835 length 87
+ Stream: column 3 section ROW_INDEX start: 10922 length 111
+ Stream: column 4 section ROW_INDEX start: 11033 length 108
+ Stream: column 5 section ROW_INDEX start: 11141 length 101
+ Stream: column 1 section DATA start: 11242 length 871
+ Stream: column 2 section DATA start: 12113 length 362
+ Stream: column 2 section LENGTH start: 12475 length 8
+ Stream: column 2 section DICTIONARY_DATA start: 12483 length 23
+ Stream: column 3 section DATA start: 12506 length 5167
+ Stream: column 4 section DATA start: 17673 length 524
+ Stream: column 4 section SECONDARY start: 18197 length 118
+ Stream: column 5 section DATA start: 18315 length 2913
+ Stream: column 5 section SECONDARY start: 21228 length 118
+ Encoding column 0: DIRECT
+ Encoding column 1: DIRECT_V2
+ Encoding column 2: DICTIONARY_V2[6]
+ Encoding column 3: DIRECT
+ Encoding column 4: DIRECT_V2
+ Encoding column 5: DIRECT_V2
+ Row group indices for column 1:
+ Entry 0: count: 10000 hasNull: false min: 2 max: 100 sum: 999815 positions: 0,0,0
+ Entry 1: count: 10000 hasNull: false min: 29 max: 100 sum: 999899 positions: 133,1071,391
+ Entry 2: count: 10000 hasNull: false min: 2 max: 100 sum: 999807 positions: 292,2147,391
+ Entry 3: count: 10000 hasNull: false min: 13 max: 100 sum: 999842 positions: 453,3223,391
+ Entry 4: count: 10000 hasNull: false min: 5 max: 100 sum: 999875 positions: 683,203,391
+
+File length: 21814 bytes
+Padding length: 0 bytes
+Padding ratio: 0%
+-- END ORC FILE DUMP --
+2 foo 0.8 1 1969-12-31 16:00:00
+PREHOOK: query: DROP TABLE orc_split_elim
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@orc_split_elim
+PREHOOK: Output: default@orc_split_elim
+PREHOOK: query: DROP TABLE orcfile_merge1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@orcfile_merge1
+PREHOOK: Output: default@orcfile_merge1
http://git-wip-us.apache.org/repos/asf/hive/blob/2d25cf23/ql/src/test/results/clientpositive/tez/orc_merge10.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/orc_merge10.q.out b/ql/src/test/results/clientpositive/tez/orc_merge10.q.out
new file mode 100644
index 0000000..d7ea13a
--- /dev/null
+++ b/ql/src/test/results/clientpositive/tez/orc_merge10.q.out
@@ -0,0 +1,709 @@
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+DROP TABLE orcfile_merge1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+DROP TABLE orcfile_merge1
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE orcfile_merge1b
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE orcfile_merge1b
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE orcfile_merge1c
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE orcfile_merge1c
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE orcfile_merge1 (key INT, value STRING)
+ PARTITIONED BY (ds STRING, part STRING) STORED AS ORC tblproperties("orc.compress"="SNAPPY","orc.compress.size"="4096")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@orcfile_merge1
+POSTHOOK: query: CREATE TABLE orcfile_merge1 (key INT, value STRING)
+ PARTITIONED BY (ds STRING, part STRING) STORED AS ORC tblproperties("orc.compress"="SNAPPY","orc.compress.size"="4096")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@orcfile_merge1
+PREHOOK: query: CREATE TABLE orcfile_merge1b (key INT, value STRING)
+ PARTITIONED BY (ds STRING, part STRING) STORED AS ORC tblproperties("orc.compress"="SNAPPY","orc.compress.size"="4096")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@orcfile_merge1b
+POSTHOOK: query: CREATE TABLE orcfile_merge1b (key INT, value STRING)
+ PARTITIONED BY (ds STRING, part STRING) STORED AS ORC tblproperties("orc.compress"="SNAPPY","orc.compress.size"="4096")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@orcfile_merge1b
+PREHOOK: query: CREATE TABLE orcfile_merge1c (key INT, value STRING)
+ PARTITIONED BY (ds STRING, part STRING) STORED AS ORC tblproperties("orc.compress"="SNAPPY","orc.compress.size"="4096")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@orcfile_merge1c
+POSTHOOK: query: CREATE TABLE orcfile_merge1c (key INT, value STRING)
+ PARTITIONED BY (ds STRING, part STRING) STORED AS ORC tblproperties("orc.compress"="SNAPPY","orc.compress.size"="4096")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@orcfile_merge1c
+PREHOOK: query: -- merge disabled
+EXPLAIN
+ INSERT OVERWRITE TABLE orcfile_merge1 PARTITION (ds='1', part)
+ SELECT key, value, PMOD(HASH(key), 2) as part
+ FROM src
+PREHOOK: type: QUERY
+POSTHOOK: query: -- merge disabled
+EXPLAIN
+ INSERT OVERWRITE TABLE orcfile_merge1 PARTITION (ds='1', part)
+ SELECT key, value, PMOD(HASH(key), 2) as part
+ FROM src
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.orcfile_merge1
+
+ Stage: Stage-2
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ partition:
+ ds 1
+ part
+ replace: true
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.orcfile_merge1
+
+ Stage: Stage-3
+ Stats-Aggr Operator
+
+PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge1 PARTITION (ds='1', part)
+ SELECT key, value, PMOD(HASH(key), 2) as part
+ FROM src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@orcfile_merge1@ds=1
+POSTHOOK: query: INSERT OVERWRITE TABLE orcfile_merge1 PARTITION (ds='1', part)
+ SELECT key, value, PMOD(HASH(key), 2) as part
+ FROM src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@orcfile_merge1@ds=1/part=0
+POSTHOOK: Output: default@orcfile_merge1@ds=1/part=1
+POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=0).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=0).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+Found 3 items
+#### A masked pattern was here ####
+PREHOOK: query: -- auto-merge slow way
+EXPLAIN
+ INSERT OVERWRITE TABLE orcfile_merge1b PARTITION (ds='1', part)
+ SELECT key, value, PMOD(HASH(key), 2) as part
+ FROM src
+PREHOOK: type: QUERY
+POSTHOOK: query: -- auto-merge slow way
+EXPLAIN
+ INSERT OVERWRITE TABLE orcfile_merge1b PARTITION (ds='1', part)
+ SELECT key, value, PMOD(HASH(key), 2) as part
+ FROM src
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-8 depends on stages: Stage-1 , consists of Stage-5, Stage-4, Stage-6
+ Stage-5
+ Stage-2 depends on stages: Stage-5, Stage-4, Stage-7
+ Stage-0 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-0
+ Stage-4
+ Stage-6
+ Stage-7 depends on stages: Stage-6
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.orcfile_merge1b
+
+ Stage: Stage-8
+ Conditional Operator
+
+ Stage: Stage-5
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
+
+ Stage: Stage-2
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ partition:
+ ds 1
+ part
+ replace: true
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.orcfile_merge1b
+
+ Stage: Stage-3
+ Stats-Aggr Operator
+
+ Stage: Stage-4
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ File Merge
+ Map Operator Tree:
+ TableScan
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.orcfile_merge1b
+
+ Stage: Stage-6
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ File Merge
+ Map Operator Tree:
+ TableScan
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.orcfile_merge1b
+
+ Stage: Stage-7
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
+
+PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge1b PARTITION (ds='1', part)
+ SELECT key, value, PMOD(HASH(key), 2) as part
+ FROM src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@orcfile_merge1b@ds=1
+POSTHOOK: query: INSERT OVERWRITE TABLE orcfile_merge1b PARTITION (ds='1', part)
+ SELECT key, value, PMOD(HASH(key), 2) as part
+ FROM src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=0
+POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=1
+POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=0).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=0).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+Found 1 items
+#### A masked pattern was here ####
+PREHOOK: query: -- auto-merge fast way
+EXPLAIN
+ INSERT OVERWRITE TABLE orcfile_merge1c PARTITION (ds='1', part)
+ SELECT key, value, PMOD(HASH(key), 2) as part
+ FROM src
+PREHOOK: type: QUERY
+POSTHOOK: query: -- auto-merge fast way
+EXPLAIN
+ INSERT OVERWRITE TABLE orcfile_merge1c PARTITION (ds='1', part)
+ SELECT key, value, PMOD(HASH(key), 2) as part
+ FROM src
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-8 depends on stages: Stage-1 , consists of Stage-5, Stage-4, Stage-6
+ Stage-5
+ Stage-2 depends on stages: Stage-5, Stage-4, Stage-7
+ Stage-0 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-0
+ Stage-4
+ Stage-6
+ Stage-7 depends on stages: Stage-6
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.orcfile_merge1c
+
+ Stage: Stage-8
+ Conditional Operator
+
+ Stage: Stage-5
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
+
+ Stage: Stage-2
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ partition:
+ ds 1
+ part
+ replace: true
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.orcfile_merge1c
+
+ Stage: Stage-3
+ Stats-Aggr Operator
+
+ Stage: Stage-4
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ File Merge
+ Merge File Operator
+ Map Operator Tree:
+ ORC File Merge Operator
+ merge level: stripe
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+
+ Stage: Stage-6
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ File Merge
+ Merge File Operator
+ Map Operator Tree:
+ ORC File Merge Operator
+ merge level: stripe
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+
+ Stage: Stage-7
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
+
+PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge1c PARTITION (ds='1', part)
+ SELECT key, value, PMOD(HASH(key), 2) as part
+ FROM src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@orcfile_merge1c@ds=1
+POSTHOOK: query: INSERT OVERWRITE TABLE orcfile_merge1c PARTITION (ds='1', part)
+ SELECT key, value, PMOD(HASH(key), 2) as part
+ FROM src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@orcfile_merge1c@ds=1/part=0
+POSTHOOK: Output: default@orcfile_merge1c@ds=1/part=1
+POSTHOOK: Lineage: orcfile_merge1c PARTITION(ds=1,part=0).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: orcfile_merge1c PARTITION(ds=1,part=0).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: orcfile_merge1c PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: orcfile_merge1c PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+Found 1 items
+#### A masked pattern was here ####
+PREHOOK: query: -- Verify
+SELECT SUM(HASH(c)) FROM (
+ SELECT TRANSFORM(*) USING 'tr \t _' AS (c)
+ FROM orcfile_merge1 WHERE ds='1'
+) t
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orcfile_merge1
+PREHOOK: Input: default@orcfile_merge1@ds=1/part=0
+PREHOOK: Input: default@orcfile_merge1@ds=1/part=1
+#### A masked pattern was here ####
+POSTHOOK: query: -- Verify
+SELECT SUM(HASH(c)) FROM (
+ SELECT TRANSFORM(*) USING 'tr \t _' AS (c)
+ FROM orcfile_merge1 WHERE ds='1'
+) t
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orcfile_merge1
+POSTHOOK: Input: default@orcfile_merge1@ds=1/part=0
+POSTHOOK: Input: default@orcfile_merge1@ds=1/part=1
+#### A masked pattern was here ####
+-21975308766
+PREHOOK: query: SELECT SUM(HASH(c)) FROM (
+ SELECT TRANSFORM(*) USING 'tr \t _' AS (c)
+ FROM orcfile_merge1b WHERE ds='1'
+) t
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orcfile_merge1b
+PREHOOK: Input: default@orcfile_merge1b@ds=1/part=0
+PREHOOK: Input: default@orcfile_merge1b@ds=1/part=1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT SUM(HASH(c)) FROM (
+ SELECT TRANSFORM(*) USING 'tr \t _' AS (c)
+ FROM orcfile_merge1b WHERE ds='1'
+) t
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orcfile_merge1b
+POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=0
+POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=1
+#### A masked pattern was here ####
+-21975308766
+PREHOOK: query: select count(*) from orcfile_merge1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orcfile_merge1
+PREHOOK: Input: default@orcfile_merge1@ds=1/part=0
+PREHOOK: Input: default@orcfile_merge1@ds=1/part=1
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from orcfile_merge1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orcfile_merge1
+POSTHOOK: Input: default@orcfile_merge1@ds=1/part=0
+POSTHOOK: Input: default@orcfile_merge1@ds=1/part=1
+#### A masked pattern was here ####
+500
+PREHOOK: query: select count(*) from orcfile_merge1b
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orcfile_merge1b
+PREHOOK: Input: default@orcfile_merge1b@ds=1/part=0
+PREHOOK: Input: default@orcfile_merge1b@ds=1/part=1
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from orcfile_merge1b
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orcfile_merge1b
+POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=0
+POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=1
+#### A masked pattern was here ####
+500
+PREHOOK: query: -- concatenate
+explain ALTER TABLE orcfile_merge1 PARTITION (ds='1', part='0') CONCATENATE
+PREHOOK: type: ALTER_PARTITION_MERGE
+POSTHOOK: query: -- concatenate
+explain ALTER TABLE orcfile_merge1 PARTITION (ds='1', part='0') CONCATENATE
+POSTHOOK: type: ALTER_PARTITION_MERGE
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+ Stage-1 depends on stages: Stage-0
+ Stage-2 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-0
+
+ Stage: Stage-1
+ Move Operator
+ tables:
+ partition:
+ ds 1
+ part 0
+ replace: true
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.orcfile_merge1
+
+ Stage: Stage-2
+ Stats-Aggr Operator
+
+PREHOOK: query: ALTER TABLE orcfile_merge1 PARTITION (ds='1', part='0') CONCATENATE
+PREHOOK: type: ALTER_PARTITION_MERGE
+PREHOOK: Input: default@orcfile_merge1
+PREHOOK: Output: default@orcfile_merge1@ds=1/part=0
+POSTHOOK: query: ALTER TABLE orcfile_merge1 PARTITION (ds='1', part='0') CONCATENATE
+POSTHOOK: type: ALTER_PARTITION_MERGE
+POSTHOOK: Input: default@orcfile_merge1
+POSTHOOK: Output: default@orcfile_merge1@ds=1/part=0
+Found 1 items
+#### A masked pattern was here ####
+PREHOOK: query: -- Verify
+SELECT SUM(HASH(c)) FROM (
+ SELECT TRANSFORM(*) USING 'tr \t _' AS (c)
+ FROM orcfile_merge1c WHERE ds='1'
+) t
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orcfile_merge1c
+PREHOOK: Input: default@orcfile_merge1c@ds=1/part=0
+PREHOOK: Input: default@orcfile_merge1c@ds=1/part=1
+#### A masked pattern was here ####
+POSTHOOK: query: -- Verify
+SELECT SUM(HASH(c)) FROM (
+ SELECT TRANSFORM(*) USING 'tr \t _' AS (c)
+ FROM orcfile_merge1c WHERE ds='1'
+) t
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orcfile_merge1c
+POSTHOOK: Input: default@orcfile_merge1c@ds=1/part=0
+POSTHOOK: Input: default@orcfile_merge1c@ds=1/part=1
+#### A masked pattern was here ####
+-21975308766
+PREHOOK: query: SELECT SUM(HASH(c)) FROM (
+ SELECT TRANSFORM(*) USING 'tr \t _' AS (c)
+ FROM orcfile_merge1 WHERE ds='1'
+) t
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orcfile_merge1
+PREHOOK: Input: default@orcfile_merge1@ds=1/part=0
+PREHOOK: Input: default@orcfile_merge1@ds=1/part=1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT SUM(HASH(c)) FROM (
+ SELECT TRANSFORM(*) USING 'tr \t _' AS (c)
+ FROM orcfile_merge1 WHERE ds='1'
+) t
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orcfile_merge1
+POSTHOOK: Input: default@orcfile_merge1@ds=1/part=0
+POSTHOOK: Input: default@orcfile_merge1@ds=1/part=1
+#### A masked pattern was here ####
+-21975308766
+PREHOOK: query: select count(*) from orcfile_merge1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orcfile_merge1
+PREHOOK: Input: default@orcfile_merge1@ds=1/part=0
+PREHOOK: Input: default@orcfile_merge1@ds=1/part=1
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from orcfile_merge1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orcfile_merge1
+POSTHOOK: Input: default@orcfile_merge1@ds=1/part=0
+POSTHOOK: Input: default@orcfile_merge1@ds=1/part=1
+#### A masked pattern was here ####
+500
+PREHOOK: query: select count(*) from orcfile_merge1c
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orcfile_merge1c
+PREHOOK: Input: default@orcfile_merge1c@ds=1/part=0
+PREHOOK: Input: default@orcfile_merge1c@ds=1/part=1
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from orcfile_merge1c
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orcfile_merge1c
+POSTHOOK: Input: default@orcfile_merge1c@ds=1/part=0
+POSTHOOK: Input: default@orcfile_merge1c@ds=1/part=1
+#### A masked pattern was here ####
+500
+PREHOOK: query: select * from orcfile_merge1 where ds='1' and part='0' limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orcfile_merge1
+PREHOOK: Input: default@orcfile_merge1@ds=1/part=0
+#### A masked pattern was here ####
+-- BEGIN ORC FILE DUMP --
+#### A masked pattern was here ####
+File Version: 0.12 with HIVE_4243
+Rows: 242
+Compression: SNAPPY
+Compression size: 4096
+Type: struct<key:int,value:string>
+
+Stripe Statistics:
+ Stripe 1:
+ Column 0: count: 90 hasNull: false
+ Column 1: count: 90 hasNull: false min: 0 max: 495 sum: 22736
+ Column 2: count: 90 hasNull: false min: val_0 max: val_86 sum: 612
+ Stripe 2:
+ Column 0: count: 78 hasNull: false
+ Column 1: count: 78 hasNull: false min: 0 max: 497 sum: 18371
+ Column 2: count: 78 hasNull: false min: val_0 max: val_95 sum: 529
+ Stripe 3:
+ Column 0: count: 74 hasNull: false
+ Column 1: count: 74 hasNull: false min: 2 max: 493 sum: 19663
+ Column 2: count: 74 hasNull: false min: val_105 max: val_97 sum: 505
+
+File Statistics:
+ Column 0: count: 242 hasNull: false
+ Column 1: count: 242 hasNull: false min: 0 max: 497 sum: 60770
+ Column 2: count: 242 hasNull: false min: val_0 max: val_97 sum: 1646
+
+Stripes:
+ Stripe: offset: 3 data: 616 rows: 90 tail: 61 index: 76
+ Stream: column 0 section ROW_INDEX start: 3 length 11
+ Stream: column 1 section ROW_INDEX start: 14 length 27
+ Stream: column 2 section ROW_INDEX start: 41 length 38
+ Stream: column 1 section DATA start: 79 length 185
+ Stream: column 2 section DATA start: 264 length 377
+ Stream: column 2 section LENGTH start: 641 length 54
+ Encoding column 0: DIRECT
+ Encoding column 1: DIRECT_V2
+ Encoding column 2: DIRECT_V2
+ Row group indices for column 1:
+ Entry 0: count: 90 hasNull: false min: 0 max: 495 sum: 22736 positions: 0,0,0
+ Stripe: offset: 756 data: 544 rows: 78 tail: 61 index: 76
+ Stream: column 0 section ROW_INDEX start: 756 length 11
+ Stream: column 1 section ROW_INDEX start: 767 length 27
+ Stream: column 2 section ROW_INDEX start: 794 length 38
+ Stream: column 1 section DATA start: 832 length 161
+ Stream: column 2 section DATA start: 993 length 332
+ Stream: column 2 section LENGTH start: 1325 length 51
+ Encoding column 0: DIRECT
+ Encoding column 1: DIRECT_V2
+ Encoding column 2: DIRECT_V2
+ Row group indices for column 1:
+ Entry 0: count: 78 hasNull: false min: 0 max: 497 sum: 18371 positions: 0,0,0
+ Stripe: offset: 1437 data: 519 rows: 74 tail: 61 index: 78
+ Stream: column 0 section ROW_INDEX start: 1437 length 11
+ Stream: column 1 section ROW_INDEX start: 1448 length 27
+ Stream: column 2 section ROW_INDEX start: 1475 length 40
+ Stream: column 1 section DATA start: 1515 length 153
+ Stream: column 2 section DATA start: 1668 length 331
+ Stream: column 2 section LENGTH start: 1999 length 35
+ Encoding column 0: DIRECT
+ Encoding column 1: DIRECT_V2
+ Encoding column 2: DIRECT_V2
+ Row group indices for column 1:
+ Entry 0: count: 74 hasNull: false min: 2 max: 493 sum: 19663 positions: 0,0,0
+
+File length: 2393 bytes
+Padding length: 0 bytes
+Padding ratio: 0%
+-- END ORC FILE DUMP --
+86 val_86 1 0
+PREHOOK: query: select * from orcfile_merge1c where ds='1' and part='0' limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orcfile_merge1c
+PREHOOK: Input: default@orcfile_merge1c@ds=1/part=0
+#### A masked pattern was here ####
+-- BEGIN ORC FILE DUMP --
+#### A masked pattern was here ####
+File Version: 0.12 with HIVE_4243
+Rows: 242
+Compression: SNAPPY
+Compression size: 4096
+Type: struct<key:int,value:string>
+
+Stripe Statistics:
+ Stripe 1:
+ Column 0: count: 90 hasNull: false
+ Column 1: count: 90 hasNull: false min: 0 max: 495 sum: 22736
+ Column 2: count: 90 hasNull: false min: val_0 max: val_86 sum: 612
+ Stripe 2:
+ Column 0: count: 78 hasNull: false
+ Column 1: count: 78 hasNull: false min: 0 max: 497 sum: 18371
+ Column 2: count: 78 hasNull: false min: val_0 max: val_95 sum: 529
+ Stripe 3:
+ Column 0: count: 74 hasNull: false
+ Column 1: count: 74 hasNull: false min: 2 max: 493 sum: 19663
+ Column 2: count: 74 hasNull: false min: val_105 max: val_97 sum: 505
+
+File Statistics:
+ Column 0: count: 242 hasNull: false
+ Column 1: count: 242 hasNull: false min: 0 max: 497 sum: 60770
+ Column 2: count: 242 hasNull: false min: val_0 max: val_97 sum: 1646
+
+Stripes:
+ Stripe: offset: 3 data: 616 rows: 90 tail: 61 index: 76
+ Stream: column 0 section ROW_INDEX start: 3 length 11
+ Stream: column 1 section ROW_INDEX start: 14 length 27
+ Stream: column 2 section ROW_INDEX start: 41 length 38
+ Stream: column 1 section DATA start: 79 length 185
+ Stream: column 2 section DATA start: 264 length 377
+ Stream: column 2 section LENGTH start: 641 length 54
+ Encoding column 0: DIRECT
+ Encoding column 1: DIRECT_V2
+ Encoding column 2: DIRECT_V2
+ Row group indices for column 1:
+ Entry 0: count: 90 hasNull: false min: 0 max: 495 sum: 22736 positions: 0,0,0
+ Stripe: offset: 756 data: 544 rows: 78 tail: 61 index: 76
+ Stream: column 0 section ROW_INDEX start: 756 length 11
+ Stream: column 1 section ROW_INDEX start: 767 length 27
+ Stream: column 2 section ROW_INDEX start: 794 length 38
+ Stream: column 1 section DATA start: 832 length 161
+ Stream: column 2 section DATA start: 993 length 332
+ Stream: column 2 section LENGTH start: 1325 length 51
+ Encoding column 0: DIRECT
+ Encoding column 1: DIRECT_V2
+ Encoding column 2: DIRECT_V2
+ Row group indices for column 1:
+ Entry 0: count: 78 hasNull: false min: 0 max: 497 sum: 18371 positions: 0,0,0
+ Stripe: offset: 1437 data: 519 rows: 74 tail: 61 index: 78
+ Stream: column 0 section ROW_INDEX start: 1437 length 11
+ Stream: column 1 section ROW_INDEX start: 1448 length 27
+ Stream: column 2 section ROW_INDEX start: 1475 length 40
+ Stream: column 1 section DATA start: 1515 length 153
+ Stream: column 2 section DATA start: 1668 length 331
+ Stream: column 2 section LENGTH start: 1999 length 35
+ Encoding column 0: DIRECT
+ Encoding column 1: DIRECT_V2
+ Encoding column 2: DIRECT_V2
+ Row group indices for column 1:
+ Entry 0: count: 74 hasNull: false min: 2 max: 493 sum: 19663 positions: 0,0,0
+
+File length: 2393 bytes
+Padding length: 0 bytes
+Padding ratio: 0%
+-- END ORC FILE DUMP --
+86 val_86 1 0
+PREHOOK: query: DROP TABLE orcfile_merge1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@orcfile_merge1
+PREHOOK: Output: default@orcfile_merge1
+PREHOOK: query: DROP TABLE orcfile_merge1b
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@orcfile_merge1b
+PREHOOK: Output: default@orcfile_merge1b
+PREHOOK: query: DROP TABLE orcfile_merge1c
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@orcfile_merge1c
+PREHOOK: Output: default@orcfile_merge1c
[02/12] hive git commit: HIVE-12417 : Support for exclamation mark
missing in regexp (Olaf Flebbe, reviewed by Sergey Shelukhin)
Posted by om...@apache.org.
HIVE-12417 : Support for exclamation mark missing in regexp (Olaf Flebbe, reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7e8fc2b3
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7e8fc2b3
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7e8fc2b3
Branch: refs/heads/master-fixed
Commit: 7e8fc2b33bc493cdf086a17cac8f8ae7eb6b6e2d
Parents: 51445d4
Author: Sergey Shelukhin <se...@apache.org>
Authored: Thu Nov 19 14:49:49 2015 -0800
Committer: Owen O'Malley <om...@apache.org>
Committed: Fri Nov 20 13:46:22 2015 -0800
----------------------------------------------------------------------
ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/7e8fc2b3/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g
index e9fbfb1..395152f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g
@@ -377,7 +377,7 @@ RegexComponent
: 'a'..'z' | 'A'..'Z' | '0'..'9' | '_'
| PLUS | STAR | QUESTION | MINUS | DOT
| LPAREN | RPAREN | LSQUARE | RSQUARE | LCURLY | RCURLY
- | BITWISEXOR | BITWISEOR | DOLLAR
+ | BITWISEXOR | BITWISEOR | DOLLAR | '!'
;
StringLiteral
[03/12] hive git commit: HIVE-12405 : Comparison bug in
HiveSplitGenerator.InputSplitComparator#compare() (Aleksei Statkevich via
Gopal V)
Posted by om...@apache.org.
HIVE-12405 : Comparison bug in HiveSplitGenerator.InputSplitComparator#compare() (Aleksei Statkevich via Gopal V)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/51445d44
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/51445d44
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/51445d44
Branch: refs/heads/master-fixed
Commit: 51445d44992889403cb9044ee677027e937ef2b0
Parents: 6563422
Author: Aleksei Statkevich <me...@gmail.com>
Authored: Thu Nov 12 23:17:00 2015 -0800
Committer: Owen O'Malley <om...@apache.org>
Committed: Fri Nov 20 13:46:22 2015 -0800
----------------------------------------------------------------------
.../hive/ql/exec/tez/HiveSplitGenerator.java | 4 +-
.../ql/exec/tez/InputSplitComparatorTest.java | 39 ++++++++++++++++++++
2 files changed, 41 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/51445d44/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java
index c370381..8902acb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java
@@ -259,7 +259,7 @@ public class HiveSplitGenerator extends InputInitializer {
}
// Descending sort based on split size| Followed by file name. Followed by startPosition.
- private static class InputSplitComparator implements Comparator<InputSplit> {
+ static class InputSplitComparator implements Comparator<InputSplit> {
@Override
public int compare(InputSplit o1, InputSplit o2) {
try {
@@ -278,7 +278,7 @@ public class HiveSplitGenerator extends InputInitializer {
// Compare start Position
long startPos1 = fs1.getStart();
long startPos2 = fs2.getStart();
- if (startPos1 > startPos1) {
+ if (startPos1 > startPos2) {
return 1;
} else if (startPos1 < startPos2) {
return -1;
http://git-wip-us.apache.org/repos/asf/hive/blob/51445d44/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/InputSplitComparatorTest.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/InputSplitComparatorTest.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/InputSplitComparatorTest.java
new file mode 100644
index 0000000..add65ae
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/InputSplitComparatorTest.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.tez;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.FileSplit;
+import org.junit.Test;
+
+import static org.apache.hadoop.hive.ql.exec.tez.HiveSplitGenerator.InputSplitComparator;
+import static org.junit.Assert.assertEquals;
+
+public class InputSplitComparatorTest {
+
+ private static final String[] EMPTY = new String[]{};
+
+ @Test
+ public void testCompare1() throws Exception {
+ FileSplit split1 = new FileSplit(new Path("/abc/def"), 2000L, 500L, EMPTY);
+ FileSplit split2 = new FileSplit(new Path("/abc/def"), 1000L, 500L, EMPTY);
+ InputSplitComparator comparator = new InputSplitComparator();
+ assertEquals(1, comparator.compare(split1, split2));
+ }
+}
[09/12] hive git commit: HIVE-12450: OrcFileMergeOperator does not
use correct compression buffer size (Prasanth Jayachandran reviewed by Sergey
Shelukhin)
Posted by om...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/2d25cf23/ql/src/test/results/clientpositive/tez/orc_merge11.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/orc_merge11.q.out b/ql/src/test/results/clientpositive/tez/orc_merge11.q.out
new file mode 100644
index 0000000..da608db
--- /dev/null
+++ b/ql/src/test/results/clientpositive/tez/orc_merge11.q.out
@@ -0,0 +1,316 @@
+PREHOOK: query: DROP TABLE orcfile_merge1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE orcfile_merge1
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE orc_split_elim
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE orc_split_elim
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table orc_split_elim (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) stored as orc
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@orc_split_elim
+POSTHOOK: query: create table orc_split_elim (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) stored as orc
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@orc_split_elim
+PREHOOK: query: load data local inpath '../../data/files/orc_split_elim.orc' into table orc_split_elim
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@orc_split_elim
+POSTHOOK: query: load data local inpath '../../data/files/orc_split_elim.orc' into table orc_split_elim
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@orc_split_elim
+PREHOOK: query: load data local inpath '../../data/files/orc_split_elim.orc' into table orc_split_elim
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@orc_split_elim
+POSTHOOK: query: load data local inpath '../../data/files/orc_split_elim.orc' into table orc_split_elim
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@orc_split_elim
+PREHOOK: query: create table orcfile_merge1 (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) stored as orc tblproperties("orc.compress.size"="4096")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@orcfile_merge1
+POSTHOOK: query: create table orcfile_merge1 (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) stored as orc tblproperties("orc.compress.size"="4096")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@orcfile_merge1
+PREHOOK: query: insert overwrite table orcfile_merge1 select * from orc_split_elim
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_split_elim
+PREHOOK: Output: default@orcfile_merge1
+POSTHOOK: query: insert overwrite table orcfile_merge1 select * from orc_split_elim
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_split_elim
+POSTHOOK: Output: default@orcfile_merge1
+POSTHOOK: Lineage: orcfile_merge1.decimal1 SIMPLE [(orc_split_elim)orc_split_elim.FieldSchema(name:decimal1, type:decimal(10,0), comment:null), ]
+POSTHOOK: Lineage: orcfile_merge1.string1 SIMPLE [(orc_split_elim)orc_split_elim.FieldSchema(name:string1, type:string, comment:null), ]
+POSTHOOK: Lineage: orcfile_merge1.subtype SIMPLE [(orc_split_elim)orc_split_elim.FieldSchema(name:subtype, type:double, comment:null), ]
+POSTHOOK: Lineage: orcfile_merge1.ts SIMPLE [(orc_split_elim)orc_split_elim.FieldSchema(name:ts, type:timestamp, comment:null), ]
+POSTHOOK: Lineage: orcfile_merge1.userid SIMPLE [(orc_split_elim)orc_split_elim.FieldSchema(name:userid, type:bigint, comment:null), ]
+PREHOOK: query: insert into table orcfile_merge1 select * from orc_split_elim
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_split_elim
+PREHOOK: Output: default@orcfile_merge1
+POSTHOOK: query: insert into table orcfile_merge1 select * from orc_split_elim
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_split_elim
+POSTHOOK: Output: default@orcfile_merge1
+POSTHOOK: Lineage: orcfile_merge1.decimal1 SIMPLE [(orc_split_elim)orc_split_elim.FieldSchema(name:decimal1, type:decimal(10,0), comment:null), ]
+POSTHOOK: Lineage: orcfile_merge1.string1 SIMPLE [(orc_split_elim)orc_split_elim.FieldSchema(name:string1, type:string, comment:null), ]
+POSTHOOK: Lineage: orcfile_merge1.subtype SIMPLE [(orc_split_elim)orc_split_elim.FieldSchema(name:subtype, type:double, comment:null), ]
+POSTHOOK: Lineage: orcfile_merge1.ts SIMPLE [(orc_split_elim)orc_split_elim.FieldSchema(name:ts, type:timestamp, comment:null), ]
+POSTHOOK: Lineage: orcfile_merge1.userid SIMPLE [(orc_split_elim)orc_split_elim.FieldSchema(name:userid, type:bigint, comment:null), ]
+Found 2 items
+#### A masked pattern was here ####
+PREHOOK: query: select * from orcfile_merge1 limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orcfile_merge1
+#### A masked pattern was here ####
+-- BEGIN ORC FILE DUMP --
+#### A masked pattern was here ####
+File Version: 0.12 with HIVE_4243
+Rows: 50000
+Compression: ZLIB
+Compression size: 4096
+Type: struct<userid:bigint,string1:string,subtype:double,decimal1:decimal(10,0),ts:timestamp>
+
+Stripe Statistics:
+ Stripe 1:
+ Column 0: count: 50000 hasNull: false
+ Column 1: count: 50000 hasNull: false min: 2 max: 100 sum: 4999238
+ Column 2: count: 50000 hasNull: false min: bar max: zebra sum: 249980
+ Column 3: count: 50000 hasNull: false min: 0.8 max: 80.0 sum: 400102.80000000005
+ Column 4: count: 50000 hasNull: false min: 0 max: 6 sum: 32
+ Column 5: count: 50000 hasNull: false min: 1969-12-31 16:00:00.0 max: 1969-12-31 16:04:10.0
+
+File Statistics:
+ Column 0: count: 50000 hasNull: false
+ Column 1: count: 50000 hasNull: false min: 2 max: 100 sum: 4999238
+ Column 2: count: 50000 hasNull: false min: bar max: zebra sum: 249980
+ Column 3: count: 50000 hasNull: false min: 0.8 max: 80.0 sum: 400102.80000000005
+ Column 4: count: 50000 hasNull: false min: 0 max: 6 sum: 32
+ Column 5: count: 50000 hasNull: false min: 1969-12-31 16:00:00.0 max: 1969-12-31 16:04:10.0
+
+Stripes:
+ Stripe: offset: 3 data: 10104 rows: 50000 tail: 117 index: 509
+ Stream: column 0 section ROW_INDEX start: 3 length 17
+ Stream: column 1 section ROW_INDEX start: 20 length 85
+ Stream: column 2 section ROW_INDEX start: 105 length 87
+ Stream: column 3 section ROW_INDEX start: 192 length 111
+ Stream: column 4 section ROW_INDEX start: 303 length 108
+ Stream: column 5 section ROW_INDEX start: 411 length 101
+ Stream: column 1 section DATA start: 512 length 871
+ Stream: column 2 section DATA start: 1383 length 362
+ Stream: column 2 section LENGTH start: 1745 length 8
+ Stream: column 2 section DICTIONARY_DATA start: 1753 length 23
+ Stream: column 3 section DATA start: 1776 length 5167
+ Stream: column 4 section DATA start: 6943 length 524
+ Stream: column 4 section SECONDARY start: 7467 length 118
+ Stream: column 5 section DATA start: 7585 length 2913
+ Stream: column 5 section SECONDARY start: 10498 length 118
+ Encoding column 0: DIRECT
+ Encoding column 1: DIRECT_V2
+ Encoding column 2: DICTIONARY_V2[6]
+ Encoding column 3: DIRECT
+ Encoding column 4: DIRECT_V2
+ Encoding column 5: DIRECT_V2
+ Row group indices for column 1:
+ Entry 0: count: 10000 hasNull: false min: 2 max: 100 sum: 999815 positions: 0,0,0
+ Entry 1: count: 10000 hasNull: false min: 29 max: 100 sum: 999899 positions: 133,1071,391
+ Entry 2: count: 10000 hasNull: false min: 2 max: 100 sum: 999807 positions: 292,2147,391
+ Entry 3: count: 10000 hasNull: false min: 13 max: 100 sum: 999842 positions: 453,3223,391
+ Entry 4: count: 10000 hasNull: false min: 5 max: 100 sum: 999875 positions: 683,203,391
+
+File length: 11071 bytes
+Padding length: 0 bytes
+Padding ratio: 0%
+-- END ORC FILE DUMP --
+-- BEGIN ORC FILE DUMP --
+#### A masked pattern was here ####
+File Version: 0.12 with HIVE_4243
+Rows: 50000
+Compression: ZLIB
+Compression size: 4096
+Type: struct<userid:bigint,string1:string,subtype:double,decimal1:decimal(10,0),ts:timestamp>
+
+Stripe Statistics:
+ Stripe 1:
+ Column 0: count: 50000 hasNull: false
+ Column 1: count: 50000 hasNull: false min: 2 max: 100 sum: 4999238
+ Column 2: count: 50000 hasNull: false min: bar max: zebra sum: 249980
+ Column 3: count: 50000 hasNull: false min: 0.8 max: 80.0 sum: 400102.80000000005
+ Column 4: count: 50000 hasNull: false min: 0 max: 6 sum: 32
+ Column 5: count: 50000 hasNull: false min: 1969-12-31 16:00:00.0 max: 1969-12-31 16:04:10.0
+
+File Statistics:
+ Column 0: count: 50000 hasNull: false
+ Column 1: count: 50000 hasNull: false min: 2 max: 100 sum: 4999238
+ Column 2: count: 50000 hasNull: false min: bar max: zebra sum: 249980
+ Column 3: count: 50000 hasNull: false min: 0.8 max: 80.0 sum: 400102.80000000005
+ Column 4: count: 50000 hasNull: false min: 0 max: 6 sum: 32
+ Column 5: count: 50000 hasNull: false min: 1969-12-31 16:00:00.0 max: 1969-12-31 16:04:10.0
+
+Stripes:
+ Stripe: offset: 3 data: 10104 rows: 50000 tail: 117 index: 509
+ Stream: column 0 section ROW_INDEX start: 3 length 17
+ Stream: column 1 section ROW_INDEX start: 20 length 85
+ Stream: column 2 section ROW_INDEX start: 105 length 87
+ Stream: column 3 section ROW_INDEX start: 192 length 111
+ Stream: column 4 section ROW_INDEX start: 303 length 108
+ Stream: column 5 section ROW_INDEX start: 411 length 101
+ Stream: column 1 section DATA start: 512 length 871
+ Stream: column 2 section DATA start: 1383 length 362
+ Stream: column 2 section LENGTH start: 1745 length 8
+ Stream: column 2 section DICTIONARY_DATA start: 1753 length 23
+ Stream: column 3 section DATA start: 1776 length 5167
+ Stream: column 4 section DATA start: 6943 length 524
+ Stream: column 4 section SECONDARY start: 7467 length 118
+ Stream: column 5 section DATA start: 7585 length 2913
+ Stream: column 5 section SECONDARY start: 10498 length 118
+ Encoding column 0: DIRECT
+ Encoding column 1: DIRECT_V2
+ Encoding column 2: DICTIONARY_V2[6]
+ Encoding column 3: DIRECT
+ Encoding column 4: DIRECT_V2
+ Encoding column 5: DIRECT_V2
+ Row group indices for column 1:
+ Entry 0: count: 10000 hasNull: false min: 2 max: 100 sum: 999815 positions: 0,0,0
+ Entry 1: count: 10000 hasNull: false min: 29 max: 100 sum: 999899 positions: 133,1071,391
+ Entry 2: count: 10000 hasNull: false min: 2 max: 100 sum: 999807 positions: 292,2147,391
+ Entry 3: count: 10000 hasNull: false min: 13 max: 100 sum: 999842 positions: 453,3223,391
+ Entry 4: count: 10000 hasNull: false min: 5 max: 100 sum: 999875 positions: 683,203,391
+
+File length: 11071 bytes
+Padding length: 0 bytes
+Padding ratio: 0%
+-- END ORC FILE DUMP --
+2 foo 0.8 1 1969-12-31 16:00:00
+PREHOOK: query: -- concatenate
+ALTER TABLE orcfile_merge1 CONCATENATE
+PREHOOK: type: ALTER_TABLE_MERGE
+PREHOOK: Input: default@orcfile_merge1
+PREHOOK: Output: default@orcfile_merge1
+Found 1 items
+#### A masked pattern was here ####
+PREHOOK: query: select count(*) from orc_split_elim
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_split_elim
+#### A masked pattern was here ####
+50000
+PREHOOK: query: -- will have double the number of rows
+select count(*) from orcfile_merge1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orcfile_merge1
+#### A masked pattern was here ####
+100000
+PREHOOK: query: select * from orcfile_merge1 limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orcfile_merge1
+#### A masked pattern was here ####
+-- BEGIN ORC FILE DUMP --
+#### A masked pattern was here ####
+File Version: 0.12 with HIVE_4243
+Rows: 100000
+Compression: ZLIB
+Compression size: 4096
+Type: struct<userid:bigint,string1:string,subtype:double,decimal1:decimal(10,0),ts:timestamp>
+
+Stripe Statistics:
+ Stripe 1:
+ Column 0: count: 50000 hasNull: false
+ Column 1: count: 50000 hasNull: false min: 2 max: 100 sum: 4999238
+ Column 2: count: 50000 hasNull: false min: bar max: zebra sum: 249980
+ Column 3: count: 50000 hasNull: false min: 0.8 max: 80.0 sum: 400102.80000000005
+ Column 4: count: 50000 hasNull: false min: 0 max: 6 sum: 32
+ Column 5: count: 50000 hasNull: false min: 1969-12-31 16:00:00.0 max: 1969-12-31 16:04:10.0
+ Stripe 2:
+ Column 0: count: 50000 hasNull: false
+ Column 1: count: 50000 hasNull: false min: 2 max: 100 sum: 4999238
+ Column 2: count: 50000 hasNull: false min: bar max: zebra sum: 249980
+ Column 3: count: 50000 hasNull: false min: 0.8 max: 80.0 sum: 400102.80000000005
+ Column 4: count: 50000 hasNull: false min: 0 max: 6 sum: 32
+ Column 5: count: 50000 hasNull: false min: 1969-12-31 16:00:00.0 max: 1969-12-31 16:04:10.0
+
+File Statistics:
+ Column 0: count: 100000 hasNull: false
+ Column 1: count: 100000 hasNull: false min: 2 max: 100 sum: 9998476
+ Column 2: count: 100000 hasNull: false min: bar max: zebra sum: 499960
+ Column 3: count: 100000 hasNull: false min: 0.8 max: 80.0 sum: 800205.6000000001
+ Column 4: count: 100000 hasNull: false min: 0 max: 6 sum: 64
+ Column 5: count: 100000 hasNull: false min: 1969-12-31 16:00:00.0 max: 1969-12-31 16:04:10.0
+
+Stripes:
+ Stripe: offset: 3 data: 10104 rows: 50000 tail: 117 index: 509
+ Stream: column 0 section ROW_INDEX start: 3 length 17
+ Stream: column 1 section ROW_INDEX start: 20 length 85
+ Stream: column 2 section ROW_INDEX start: 105 length 87
+ Stream: column 3 section ROW_INDEX start: 192 length 111
+ Stream: column 4 section ROW_INDEX start: 303 length 108
+ Stream: column 5 section ROW_INDEX start: 411 length 101
+ Stream: column 1 section DATA start: 512 length 871
+ Stream: column 2 section DATA start: 1383 length 362
+ Stream: column 2 section LENGTH start: 1745 length 8
+ Stream: column 2 section DICTIONARY_DATA start: 1753 length 23
+ Stream: column 3 section DATA start: 1776 length 5167
+ Stream: column 4 section DATA start: 6943 length 524
+ Stream: column 4 section SECONDARY start: 7467 length 118
+ Stream: column 5 section DATA start: 7585 length 2913
+ Stream: column 5 section SECONDARY start: 10498 length 118
+ Encoding column 0: DIRECT
+ Encoding column 1: DIRECT_V2
+ Encoding column 2: DICTIONARY_V2[6]
+ Encoding column 3: DIRECT
+ Encoding column 4: DIRECT_V2
+ Encoding column 5: DIRECT_V2
+ Row group indices for column 1:
+ Entry 0: count: 10000 hasNull: false min: 2 max: 100 sum: 999815 positions: 0,0,0
+ Entry 1: count: 10000 hasNull: false min: 29 max: 100 sum: 999899 positions: 133,1071,391
+ Entry 2: count: 10000 hasNull: false min: 2 max: 100 sum: 999807 positions: 292,2147,391
+ Entry 3: count: 10000 hasNull: false min: 13 max: 100 sum: 999842 positions: 453,3223,391
+ Entry 4: count: 10000 hasNull: false min: 5 max: 100 sum: 999875 positions: 683,203,391
+ Stripe: offset: 10733 data: 10104 rows: 50000 tail: 117 index: 509
+ Stream: column 0 section ROW_INDEX start: 10733 length 17
+ Stream: column 1 section ROW_INDEX start: 10750 length 85
+ Stream: column 2 section ROW_INDEX start: 10835 length 87
+ Stream: column 3 section ROW_INDEX start: 10922 length 111
+ Stream: column 4 section ROW_INDEX start: 11033 length 108
+ Stream: column 5 section ROW_INDEX start: 11141 length 101
+ Stream: column 1 section DATA start: 11242 length 871
+ Stream: column 2 section DATA start: 12113 length 362
+ Stream: column 2 section LENGTH start: 12475 length 8
+ Stream: column 2 section DICTIONARY_DATA start: 12483 length 23
+ Stream: column 3 section DATA start: 12506 length 5167
+ Stream: column 4 section DATA start: 17673 length 524
+ Stream: column 4 section SECONDARY start: 18197 length 118
+ Stream: column 5 section DATA start: 18315 length 2913
+ Stream: column 5 section SECONDARY start: 21228 length 118
+ Encoding column 0: DIRECT
+ Encoding column 1: DIRECT_V2
+ Encoding column 2: DICTIONARY_V2[6]
+ Encoding column 3: DIRECT
+ Encoding column 4: DIRECT_V2
+ Encoding column 5: DIRECT_V2
+ Row group indices for column 1:
+ Entry 0: count: 10000 hasNull: false min: 2 max: 100 sum: 999815 positions: 0,0,0
+ Entry 1: count: 10000 hasNull: false min: 29 max: 100 sum: 999899 positions: 133,1071,391
+ Entry 2: count: 10000 hasNull: false min: 2 max: 100 sum: 999807 positions: 292,2147,391
+ Entry 3: count: 10000 hasNull: false min: 13 max: 100 sum: 999842 positions: 453,3223,391
+ Entry 4: count: 10000 hasNull: false min: 5 max: 100 sum: 999875 positions: 683,203,391
+
+File length: 21814 bytes
+Padding length: 0 bytes
+Padding ratio: 0%
+-- END ORC FILE DUMP --
+2 foo 0.8 1 1969-12-31 16:00:00
+PREHOOK: query: DROP TABLE orc_split_elim
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@orc_split_elim
+PREHOOK: Output: default@orc_split_elim
+PREHOOK: query: DROP TABLE orcfile_merge1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@orcfile_merge1
+PREHOOK: Output: default@orcfile_merge1
[12/12] hive git commit: HIVE-12437: SMB join in tez fails when one
of the tables is empty (Vikram Dixit K, reviewed by Siddharth Seth)
Posted by om...@apache.org.
HIVE-12437: SMB join in tez fails when one of the tables is empty (Vikram Dixit K, reviewed by Siddharth Seth)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4a13e5fb
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4a13e5fb
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4a13e5fb
Branch: refs/heads/master-fixed
Commit: 4a13e5fb24837820cb6cdd1093bc8920c639b122
Parents: 39bd58b
Author: vikram <vi...@hortonworks.com>
Authored: Fri Nov 20 13:14:58 2015 -0800
Committer: Owen O'Malley <om...@apache.org>
Committed: Fri Nov 20 13:46:23 2015 -0800
----------------------------------------------------------------------
.../hive/ql/exec/tez/CustomPartitionVertex.java | 9 +-
.../hive/ql/exec/tez/MapRecordProcessor.java | 14 +-
.../test/queries/clientpositive/tez_smb_empty.q | 13 ++
.../clientpositive/tez/tez_smb_empty.q.out | 168 +++++++++++++++++++
4 files changed, 200 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/4a13e5fb/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java
index fe1ef37..e9c14b1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java
@@ -319,9 +319,9 @@ public class CustomPartitionVertex extends VertexManagerPlugin {
Multimap<Integer, InputSplit> bucketToGroupedSplitMap) throws IOException {
// the bucket to task map should have been setup by the big table.
LOG.info("Processing events for input " + inputName);
- if (bucketToTaskMap.isEmpty()) {
- LOG.info("We don't have a routing table yet. Will need to wait for the main input"
- + " initialization");
+ if (inputNameInputSpecMap.get(mainWorkName) == null) {
+ LOG.info("We don't have a routing table yet. Will need to wait for the main input "
+ + mainWorkName + " initialization");
inputToGroupedSplitMap.put(inputName, bucketToGroupedSplitMap);
return;
}
@@ -351,6 +351,9 @@ public class CustomPartitionVertex extends VertexManagerPlugin {
for (Entry<Integer, Collection<ByteBuffer>> entry : bucketToSerializedSplitMap.asMap().entrySet()) {
Collection<Integer> destTasks = bucketToTaskMap.get(entry.getKey());
+ if ((destTasks == null) || (destTasks.isEmpty())) {
+ continue;
+ }
for (Integer task : destTasks) {
int count = 0;
for (ByteBuffer buf : entry.getValue()) {
http://git-wip-us.apache.org/repos/asf/hive/blob/4a13e5fb/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java
index 1d645a0..914b4e7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java
@@ -201,7 +201,19 @@ public class MapRecordProcessor extends RecordProcessor {
jconf.set(Utilities.INPUT_NAME, mergeMapWork.getName());
mergeMapOp.initialize(jconf, null);
// if there are no files/partitions to read, we need to skip trying to read
- boolean skipRead = mergeMapOp.getConf().getPathToAliases().isEmpty();
+ MultiMRInput multiMRInput = multiMRInputMap.get(mergeMapWork.getName());
+ boolean skipRead = false;
+ if (multiMRInput == null) {
+ l4j.info("Multi MR Input for work " + mergeMapWork.getName() + " is null. Skipping read.");
+ skipRead = true;
+ } else {
+ Collection<KeyValueReader> keyValueReaders = multiMRInput.getKeyValueReaders();
+ if ((keyValueReaders == null) || (keyValueReaders.isEmpty())) {
+ l4j.info("Key value readers are null or empty and hence skipping read. "
+ + "KeyValueReaders = " + keyValueReaders);
+ skipRead = true;
+ }
+ }
if (skipRead) {
List<Operator<?>> children = new ArrayList<Operator<?>>();
children.addAll(mergeMapOp.getConf().getAliasToWork().values());
http://git-wip-us.apache.org/repos/asf/hive/blob/4a13e5fb/ql/src/test/queries/clientpositive/tez_smb_empty.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/tez_smb_empty.q b/ql/src/test/queries/clientpositive/tez_smb_empty.q
index 196cc97..2427377 100644
--- a/ql/src/test/queries/clientpositive/tez_smb_empty.q
+++ b/ql/src/test/queries/clientpositive/tez_smb_empty.q
@@ -53,3 +53,16 @@ explain
select count(*) from tab s1 left outer join empty s2 on s1.key=s2.key join tab s3 on s1.key = s3.key;
select count(*) from tab s1 left outer join empty s2 on s1.key=s2.key join tab s3 on s1.key = s3.key;
+
+explain
+select count(*) from empty s1 join empty s3 on s1.key=s3.key;
+
+select count(*) from empty s1 join empty s3 on s1.key=s3.key;
+
+set hive.auto.convert.sortmerge.join.bigtable.selection.policy = org.apache.hadoop.hive.ql.optimizer.LeftmostBigTableSelectorForAutoSMJ;
+
+explain
+select count(*) from empty s1 join tab s3 on s1.key=s3.key;
+
+select count(*) from empty s1 join tab s3 on s1.key=s3.key;
+
http://git-wip-us.apache.org/repos/asf/hive/blob/4a13e5fb/ql/src/test/results/clientpositive/tez/tez_smb_empty.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/tez_smb_empty.q.out b/ql/src/test/results/clientpositive/tez/tez_smb_empty.q.out
index 82ec31d..a32fb12 100644
--- a/ql/src/test/results/clientpositive/tez/tez_smb_empty.q.out
+++ b/ql/src/test/results/clientpositive/tez/tez_smb_empty.q.out
@@ -674,3 +674,171 @@ POSTHOOK: Input: default@tab
POSTHOOK: Input: default@tab@ds=2008-04-08
#### A masked pattern was here ####
480
+PREHOOK: query: explain
+select count(*) from empty s1 join empty s3 on s1.key=s3.key
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(*) from empty s1 join empty s3 on s1.key=s3.key
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: s1
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Map Operator Tree:
+ TableScan
+ alias: s1
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Reducer 2
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(*) from empty s1 join empty s3 on s1.key=s3.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@empty
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from empty s1 join empty s3 on s1.key=s3.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@empty
+#### A masked pattern was here ####
+0
+PREHOOK: query: explain
+select count(*) from empty s1 join tab s3 on s1.key=s3.key
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(*) from empty s1 join tab s3 on s1.key=s3.key
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: s3
+ Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE
+ Map Operator Tree:
+ TableScan
+ alias: s1
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 key (type: int)
+ 1 key (type: int)
+ Statistics: Num rows: 133 Data size: 1411 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Reducer 2
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(*) from empty s1 join tab s3 on s1.key=s3.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@empty
+PREHOOK: Input: default@tab
+PREHOOK: Input: default@tab@ds=2008-04-08
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from empty s1 join tab s3 on s1.key=s3.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@empty
+POSTHOOK: Input: default@tab
+POSTHOOK: Input: default@tab@ds=2008-04-08
+#### A masked pattern was here ####
+0
[08/12] hive git commit: HIVE-11684: Implement limit pushdown through
outer join in CBO (Jesus Camacho Rodriguez,
reviewed by Laljo John Pullokkaran)
Posted by om...@apache.org.
HIVE-11684: Implement limit pushdown through outer join in CBO (Jesus Camacho Rodriguez, reviewed by Laljo John Pullokkaran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/39bd58b2
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/39bd58b2
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/39bd58b2
Branch: refs/heads/master-fixed
Commit: 39bd58b2f6c1614bc8f2eb9a28e0d039c7df2f73
Parents: 2d25cf2
Author: Jesus Camacho Rodriguez <jc...@apache.org>
Authored: Mon Nov 16 11:32:21 2015 +0100
Committer: Owen O'Malley <om...@apache.org>
Committed: Fri Nov 20 13:46:23 2015 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 11 +
.../test/resources/testconfiguration.properties | 1 +
.../calcite/reloperators/HiveSortLimit.java | 15 +-
.../rules/HiveProjectSortTransposeRule.java | 66 ++
.../calcite/rules/HiveSortJoinReduceRule.java | 135 +++
.../calcite/rules/HiveSortMergeRule.java | 78 ++
.../rules/HiveSortProjectTransposeRule.java | 79 ++
.../calcite/rules/HiveSortRemoveRule.java | 79 ++
.../calcite/stats/HiveRelMdRowCount.java | 18 +
.../calcite/stats/HiveRelMdSelectivity.java | 10 +
.../hadoop/hive/ql/parse/CalcitePlanner.java | 32 +-
.../clientpositive/limit_join_transpose.q | 99 +++
.../clientpositive/tez_dynpart_hashjoin_3.q | 24 +
.../clientpositive/join32_lessSize.q.out | 46 +-
.../results/clientpositive/join_merging.q.out | 34 +-
.../clientpositive/limit_join_transpose.q.out | 869 +++++++++++++++++++
.../clientpositive/spark/join32_lessSize.q.out | 122 +--
.../clientpositive/spark/join_merging.q.out | 76 +-
.../results/clientpositive/subquery_notin.q.out | 328 ++++---
.../subquery_notin_having.q.java1.7.out | 94 +-
.../clientpositive/tez/explainuser_1.q.out | 171 ++--
.../tez/tez_dynpart_hashjoin_3.q.out | 200 +++++
22 files changed, 2137 insertions(+), 450 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/39bd58b2/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 4f8209a..09f3c3e 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1311,6 +1311,17 @@ public class HiveConf extends Configuration {
"we are increasing the number of files possibly by a big margin. So, we merge aggressively."),
HIVEOPTCORRELATION("hive.optimize.correlation", false, "exploit intra-query correlations."),
+ HIVE_OPTIMIZE_LIMIT_JOIN_TRANSPOSE("hive.optimize.limitjointranspose", false,
+ "Whether to push a limit through left/right outer join. If the value is true and the size of the outer\n" +
+ "input is reduced enough (as specified in hive.optimize.limitjointranspose.reduction), the limit is pushed\n" +
+ "to the outer input; to remain semantically correct, the limit is kept on top of the join too."),
+ HIVE_OPTIMIZE_LIMIT_JOIN_TRANSPOSE_REDUCTION_PERCENTAGE("hive.optimize.limitjointranspose.reductionpercentage", 1.0f,
+ "When hive.optimize.limitjointranspose is true, this variable specifies the minimal reduction of the\n" +
+ "size of the outer input of the join that we should get in order to apply the rule."),
+ HIVE_OPTIMIZE_LIMIT_JOIN_TRANSPOSE_REDUCTION_TUPLES("hive.optimize.limitjointranspose.reductiontuples", (long) 0,
+ "When hive.optimize.limitjointranspose is true, this variable specifies the minimal reduction in the\n" +
+ "number of tuples of the outer input of the join that you should get in order to apply the rule."),
+
HIVE_OPTIMIZE_SKEWJOIN_COMPILETIME("hive.optimize.skewjoin.compiletime", false,
"Whether to create a separate plan for skewed keys for the tables in the join.\n" +
"This is based on the skewed keys stored in the metadata. At compile time, the plan is broken\n" +
http://git-wip-us.apache.org/repos/asf/hive/blob/39bd58b2/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index b86dd19..0c5140c 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -388,6 +388,7 @@ minitez.query.files=bucket_map_join_tez1.q,\
tez_insert_overwrite_local_directory_1.q,\
tez_dynpart_hashjoin_1.q,\
tez_dynpart_hashjoin_2.q,\
+ tez_dynpart_hashjoin_3.q,\
tez_vector_dynpart_hashjoin_1.q,\
tez_vector_dynpart_hashjoin_2.q,\
tez_join_hash.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/39bd58b2/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSortLimit.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSortLimit.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSortLimit.java
index 82fdc0e..3077177 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSortLimit.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSortLimit.java
@@ -44,6 +44,8 @@ public class HiveSortLimit extends Sort implements HiveRelNode {
// 4. This is used by ASTConverter after we are done with Calcite Planning
private ImmutableMap<Integer, RexNode> mapOfInputRefToRexCall;
+ private boolean ruleCreated;
+
public HiveSortLimit(RelOptCluster cluster, RelTraitSet traitSet, RelNode child,
RelCollation collation, RexNode offset, RexNode fetch) {
super(cluster, TraitsUtil.getSortTraitSet(cluster, traitSet, collation), child, collation,
@@ -74,7 +76,10 @@ public class HiveSortLimit extends Sort implements HiveRelNode {
// TODO: can we blindly copy sort trait? What if inputs changed and we
// are now sorting by different cols
RelCollation canonizedCollation = traitSet.canonize(newCollation);
- return new HiveSortLimit(getCluster(), traitSet, newInput, canonizedCollation, offset, fetch);
+ HiveSortLimit sortLimit =
+ new HiveSortLimit(getCluster(), traitSet, newInput, canonizedCollation, offset, fetch);
+ sortLimit.setRuleCreated(ruleCreated);
+ return sortLimit;
}
public RexNode getFetchExpr() {
@@ -93,6 +98,14 @@ public class HiveSortLimit extends Sort implements HiveRelNode {
public void implement(Implementor implementor) {
}
+ public boolean isRuleCreated() {
+ return ruleCreated;
+ }
+
+ public void setRuleCreated(boolean ruleCreated) {
+ this.ruleCreated = ruleCreated;
+ }
+
private static class HiveSortRelFactory implements RelFactories.SortFactory {
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/39bd58b2/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveProjectSortTransposeRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveProjectSortTransposeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveProjectSortTransposeRule.java
new file mode 100644
index 0000000..9d74b19
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveProjectSortTransposeRule.java
@@ -0,0 +1,66 @@
+package org.apache.hadoop.hive.ql.optimizer.calcite.rules;
+
+import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.plan.RelOptUtil;
+import org.apache.calcite.rel.RelCollation;
+import org.apache.calcite.rel.RelCollationTraitDef;
+import org.apache.calcite.rel.RelFieldCollation;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rex.RexUtil;
+import org.apache.calcite.util.mapping.Mappings;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortLimit;
+
+import com.google.common.collect.ImmutableList;
+
+public class HiveProjectSortTransposeRule extends RelOptRule {
+
+ public static final HiveProjectSortTransposeRule INSTANCE =
+ new HiveProjectSortTransposeRule();
+
+ //~ Constructors -----------------------------------------------------------
+
+ /**
+ * Creates a HiveProjectSortTransposeRule.
+ */
+ private HiveProjectSortTransposeRule() {
+ super(
+ operand(
+ HiveProject.class,
+ operand(HiveSortLimit.class, any())));
+ }
+
+ //~ Methods ----------------------------------------------------------------
+
+ // implement RelOptRule
+ public void onMatch(RelOptRuleCall call) {
+ final HiveProject project = call.rel(0);
+ final HiveSortLimit sort = call.rel(1);
+
+ // Determine mapping between project input and output fields. If sort
+ // relies on non-trivial expressions, we can't push.
+ final Mappings.TargetMapping map =
+ RelOptUtil.permutation(
+ project.getProjects(), project.getInput().getRowType()).inverse();
+ for (RelFieldCollation fc : sort.getCollation().getFieldCollations()) {
+ if (map.getTarget(fc.getFieldIndex()) < 0) {
+ return;
+ }
+ }
+
+ // Create new collation
+ final RelCollation newCollation =
+ RelCollationTraitDef.INSTANCE.canonize(
+ RexUtil.apply(map, sort.getCollation()));
+
+ // New operators
+ final RelNode newProject = project.copy(sort.getInput().getTraitSet(),
+ ImmutableList.<RelNode>of(sort.getInput()));
+ final HiveSortLimit newSort = sort.copy(newProject.getTraitSet(),
+ newProject, newCollation, sort.offset, sort.fetch);
+
+ call.transformTo(newSort);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/39bd58b2/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortJoinReduceRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortJoinReduceRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortJoinReduceRule.java
new file mode 100644
index 0000000..d8b9c54
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortJoinReduceRule.java
@@ -0,0 +1,135 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.optimizer.calcite.rules;
+
+import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.rel.RelCollation;
+import org.apache.calcite.rel.RelCollationTraitDef;
+import org.apache.calcite.rel.RelCollations;
+import org.apache.calcite.rel.RelFieldCollation;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.JoinRelType;
+import org.apache.calcite.rel.metadata.RelMetadataQuery;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortLimit;
+
+/**
+ * Planner rule that pushes
+ * a {@link org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortLimit}
+ * past a {@link org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin}.
+ */
+public class HiveSortJoinReduceRule extends RelOptRule {
+
+ public static final HiveSortJoinReduceRule INSTANCE =
+ new HiveSortJoinReduceRule();
+
+ //~ Constructors -----------------------------------------------------------
+
+ private HiveSortJoinReduceRule() {
+ super(
+ operand(
+ HiveSortLimit.class,
+ operand(HiveJoin.class, any())));
+ }
+
+ //~ Methods ----------------------------------------------------------------
+
+ @Override
+ public boolean matches(RelOptRuleCall call) {
+ final HiveSortLimit sortLimit = call.rel(0);
+ final HiveJoin join = call.rel(1);
+
+ // If sort does not contain a limit operation, we bail out
+ if (!HiveCalciteUtil.limitRelNode(sortLimit)) {
+ return false;
+ }
+
+ // 1) If join is not a left or right outer, we bail out
+ // 2) If any sort column is not part of the input where the
+ // sort is pushed, we bail out
+ RelNode reducedInput;
+ if (join.getJoinType() == JoinRelType.LEFT) {
+ reducedInput = join.getLeft();
+ if (sortLimit.getCollation() != RelCollations.EMPTY) {
+ for (RelFieldCollation relFieldCollation
+ : sortLimit.getCollation().getFieldCollations()) {
+ if (relFieldCollation.getFieldIndex()
+ >= join.getLeft().getRowType().getFieldCount()) {
+ return false;
+ }
+ }
+ }
+ } else if (join.getJoinType() == JoinRelType.RIGHT) {
+ reducedInput = join.getRight();
+ if (sortLimit.getCollation() != RelCollations.EMPTY) {
+ for (RelFieldCollation relFieldCollation
+ : sortLimit.getCollation().getFieldCollations()) {
+ if (relFieldCollation.getFieldIndex()
+ < join.getLeft().getRowType().getFieldCount()) {
+ return false;
+ }
+ }
+ }
+ } else {
+ return false;
+ }
+
+ // Finally, if we do not reduce the input size, we bail out
+ if (RexLiteral.intValue(sortLimit.fetch)
+ >= RelMetadataQuery.getRowCount(reducedInput)) {
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public void onMatch(RelOptRuleCall call) {
+ final HiveSortLimit sortLimit = call.rel(0);
+ final HiveJoin join = call.rel(1);
+ RelNode inputLeft = join.getLeft();
+ RelNode inputRight = join.getRight();
+
+ // We create a new sort operator on the corresponding input
+ if (join.getJoinType() == JoinRelType.LEFT) {
+ inputLeft = sortLimit.copy(sortLimit.getTraitSet(), inputLeft,
+ sortLimit.getCollation(), sortLimit.offset, sortLimit.fetch);
+ ((HiveSortLimit) inputLeft).setRuleCreated(true);
+ } else {
+ // Adjust right collation
+ final RelCollation rightCollation =
+ RelCollationTraitDef.INSTANCE.canonize(
+ RelCollations.shift(sortLimit.getCollation(),
+ -join.getLeft().getRowType().getFieldCount()));
+ inputRight = sortLimit.copy(sortLimit.getTraitSet().replace(rightCollation), inputRight,
+ rightCollation, sortLimit.offset, sortLimit.fetch);
+ ((HiveSortLimit) inputRight).setRuleCreated(true);
+ }
+ // We copy the join and the top sort operator
+ RelNode result = join.copy(join.getTraitSet(), join.getCondition(), inputLeft,
+ inputRight, join.getJoinType(), join.isSemiJoinDone());
+ result = sortLimit.copy(sortLimit.getTraitSet(), result, sortLimit.getCollation(),
+ sortLimit.offset, sortLimit.fetch);
+
+ call.transformTo(result);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/39bd58b2/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortMergeRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortMergeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortMergeRule.java
new file mode 100644
index 0000000..ebb20c2
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortMergeRule.java
@@ -0,0 +1,78 @@
+package org.apache.hadoop.hive.ql.optimizer.calcite.rules;
+
+import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortLimit;
+
+/**
+ * This rule will merge two HiveSortLimit operators.
+ *
+ * It is applied when the top match is a pure limit operation (no sorting).
+ *
+ * If the bottom operator is not synthetic and does not contain a limit,
+ * we currently bail out. Thus, we avoid a lot of unnecessary limit operations
+ * in the middle of the execution plan that could create performance regressions.
+ */
+public class HiveSortMergeRule extends RelOptRule {
+
+ public static final HiveSortMergeRule INSTANCE =
+ new HiveSortMergeRule();
+
+ //~ Constructors -----------------------------------------------------------
+
+ /**
+ * Creates a HiveSortProjectTransposeRule.
+ */
+ private HiveSortMergeRule() {
+ super(
+ operand(
+ HiveSortLimit.class,
+ operand(HiveSortLimit.class, any())));
+ }
+
+ //~ Methods ----------------------------------------------------------------
+
+ @Override
+ public boolean matches(RelOptRuleCall call) {
+ final HiveSortLimit topSortLimit = call.rel(0);
+ final HiveSortLimit bottomSortLimit = call.rel(1);
+
+ // If top operator is not a pure limit, we bail out
+ if (!HiveCalciteUtil.pureLimitRelNode(topSortLimit)) {
+ return false;
+ }
+
+ // If the bottom operator is not synthetic and it does not contain a limit,
+ // we will bail out; we do not want to end up with limits all over the tree
+ if (topSortLimit.isRuleCreated() && !bottomSortLimit.isRuleCreated() &&
+ bottomSortLimit.fetch == null) {
+ return false;
+ }
+
+ return true;
+ }
+
+ // implement RelOptRule
+ public void onMatch(RelOptRuleCall call) {
+ final HiveSortLimit topSortLimit = call.rel(0);
+ final HiveSortLimit bottomSortLimit = call.rel(1);
+
+ // Lowest limit
+ final RexNode newLimit;
+ if (bottomSortLimit.fetch != null && RexLiteral.intValue(topSortLimit.fetch)
+ >= RexLiteral.intValue(bottomSortLimit.fetch)) {
+ newLimit = bottomSortLimit.fetch;
+ } else {
+ newLimit = topSortLimit.fetch;
+ }
+
+ final HiveSortLimit newSort = bottomSortLimit.copy(bottomSortLimit.getTraitSet(),
+ bottomSortLimit.getInput(), bottomSortLimit.collation, null, newLimit);
+
+ call.transformTo(newSort);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/39bd58b2/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortProjectTransposeRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortProjectTransposeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortProjectTransposeRule.java
new file mode 100644
index 0000000..debfbe0
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortProjectTransposeRule.java
@@ -0,0 +1,79 @@
+package org.apache.hadoop.hive.ql.optimizer.calcite.rules;
+
+import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.plan.RelOptUtil;
+import org.apache.calcite.rel.RelCollation;
+import org.apache.calcite.rel.RelCollationTraitDef;
+import org.apache.calcite.rel.RelFieldCollation;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rex.RexUtil;
+import org.apache.calcite.util.mapping.Mappings;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortLimit;
+
+import com.google.common.collect.ImmutableList;
+
+public class HiveSortProjectTransposeRule extends RelOptRule {
+
+ public static final HiveSortProjectTransposeRule INSTANCE =
+ new HiveSortProjectTransposeRule();
+
+ //~ Constructors -----------------------------------------------------------
+
+ /**
+ * Creates a HiveSortProjectTransposeRule.
+ */
+ private HiveSortProjectTransposeRule() {
+ super(
+ operand(
+ HiveSortLimit.class,
+ operand(HiveProject.class, any())));
+ }
+
+ //~ Methods ----------------------------------------------------------------
+
+ @Override
+ public boolean matches(RelOptRuleCall call) {
+ final HiveSortLimit sortLimit = call.rel(0);
+
+ // If does not contain a limit operation, we bail out
+ if (!HiveCalciteUtil.limitRelNode(sortLimit)) {
+ return false;
+ }
+
+ return true;
+ }
+
+ // implement RelOptRule
+ public void onMatch(RelOptRuleCall call) {
+ final HiveSortLimit sort = call.rel(0);
+ final HiveProject project = call.rel(1);
+
+ // Determine mapping between project input and output fields. If sort
+ // relies on non-trivial expressions, we can't push.
+ final Mappings.TargetMapping map =
+ RelOptUtil.permutation(
+ project.getProjects(), project.getInput().getRowType());
+ for (RelFieldCollation fc : sort.getCollation().getFieldCollations()) {
+ if (map.getTargetOpt(fc.getFieldIndex()) < 0) {
+ return;
+ }
+ }
+
+ // Create new collation
+ final RelCollation newCollation =
+ RelCollationTraitDef.INSTANCE.canonize(
+ RexUtil.apply(map, sort.getCollation()));
+
+ // New operators
+ final HiveSortLimit newSort = sort.copy(sort.getTraitSet().replace(newCollation),
+ project.getInput(), newCollation, sort.offset, sort.fetch);
+ final RelNode newProject = project.copy(sort.getTraitSet(),
+ ImmutableList.<RelNode>of(newSort));
+
+ call.transformTo(newProject);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/39bd58b2/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortRemoveRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortRemoveRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortRemoveRule.java
new file mode 100644
index 0000000..618c717
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortRemoveRule.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.optimizer.calcite.rules;
+
+import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.plan.RelOptRuleOperand;
+import org.apache.calcite.rel.metadata.RelMetadataQuery;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortLimit;
+
+/**
+ * Planner rule that removes
+ * a {@link org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortLimit}.
+ */
+public class HiveSortRemoveRule extends RelOptRule {
+
+ protected final float reductionProportion;
+ protected final float reductionTuples;
+
+ //~ Constructors -----------------------------------------------------------
+
+ public HiveSortRemoveRule(float reductionProportion, long reductionTuples) {
+ this(operand(HiveSortLimit.class, any()), reductionProportion, reductionTuples);
+ }
+
+ private HiveSortRemoveRule(RelOptRuleOperand operand, float reductionProportion,
+ long reductionTuples) {
+ super(operand);
+ this.reductionProportion = reductionProportion;
+ this.reductionTuples = reductionTuples;
+ }
+
+ //~ Methods ----------------------------------------------------------------
+
+ @Override
+ public boolean matches(RelOptRuleCall call) {
+ final HiveSortLimit sortLimit = call.rel(0);
+
+ // If it is not created by HiveSortJoinReduceRule, we cannot remove it
+ if (!sortLimit.isRuleCreated()) {
+ return false;
+ }
+
+ // Finally, if we do not reduce the size input enough, we bail out
+ int limit = RexLiteral.intValue(sortLimit.fetch);
+ Double rowCount = RelMetadataQuery.getRowCount(sortLimit.getInput());
+ if (rowCount != null && limit <= reductionProportion * rowCount &&
+ rowCount - limit >= reductionTuples) {
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public void onMatch(RelOptRuleCall call) {
+ final HiveSortLimit sortLimit = call.rel(0);
+
+ // We remove the limit operator
+ call.transformTo(sortLimit.getInput());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/39bd58b2/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdRowCount.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdRowCount.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdRowCount.java
index 728c5aa..caf8978 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdRowCount.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdRowCount.java
@@ -30,6 +30,7 @@ import org.apache.calcite.rel.core.Join;
import org.apache.calcite.rel.core.JoinRelType;
import org.apache.calcite.rel.core.Project;
import org.apache.calcite.rel.core.SemiJoin;
+import org.apache.calcite.rel.core.Sort;
import org.apache.calcite.rel.core.TableScan;
import org.apache.calcite.rel.metadata.ReflectiveRelMetadataProvider;
import org.apache.calcite.rel.metadata.RelMdRowCount;
@@ -38,6 +39,7 @@ import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLiteral;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexUtil;
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
@@ -87,6 +89,22 @@ public class HiveRelMdRowCount extends RelMdRowCount {
return super.getRowCount(rel);
}
+ @Override
+ public Double getRowCount(Sort rel) {
+ final Double rowCount = RelMetadataQuery.getRowCount(rel.getInput());
+ if (rowCount != null && rel.fetch != null) {
+ final int offset = rel.offset == null ? 0 : RexLiteral.intValue(rel.offset);
+ final int limit = RexLiteral.intValue(rel.fetch);
+ final Double offsetLimit = new Double(offset + limit);
+ // offsetLimit is smaller than rowCount of the input operator
+ // thus, we return the offsetLimit
+ if (offsetLimit < rowCount) {
+ return offsetLimit;
+ }
+ }
+ return rowCount;
+ }
+
static class PKFKRelationInfo {
public final int fkSide;
public final double ndvScalingFactor;
http://git-wip-us.apache.org/repos/asf/hive/blob/39bd58b2/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java
index 715f24f..a0eb83d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java
@@ -61,6 +61,16 @@ public class HiveRelMdSelectivity extends RelMdSelectivity {
public Double getSelectivity(HiveJoin j, RexNode predicate) throws CalciteSemanticException {
if (j.getJoinType().equals(JoinRelType.INNER)) {
return computeInnerJoinSelectivity(j, predicate);
+ } else if (j.getJoinType().equals(JoinRelType.LEFT) ||
+ j.getJoinType().equals(JoinRelType.RIGHT)) {
+ double left = RelMetadataQuery.getRowCount(j.getLeft());
+ double right = RelMetadataQuery.getRowCount(j.getRight());
+ double product = left * right;
+ double innerJoinSelectivity = computeInnerJoinSelectivity(j, predicate);
+ if (j.getJoinType().equals(JoinRelType.LEFT)) {
+ return Math.max(innerJoinSelectivity, left/product);
+ }
+ return Math.max(innerJoinSelectivity, right/product);
}
return 1.0;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/39bd58b2/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index de67b54..bfe4633 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -148,8 +148,13 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveJoinToMultiJoinRule
import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HivePartitionPruneRule;
import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HivePreFilteringRule;
import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveProjectMergeRule;
+import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveProjectSortTransposeRule;
import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveRelFieldTrimmer;
import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveRulesRegistry;
+import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveSortJoinReduceRule;
+import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveSortMergeRule;
+import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveSortProjectTransposeRule;
+import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveSortRemoveRule;
import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveWindowingFixRule;
import org.apache.hadoop.hive.ql.optimizer.calcite.translator.ASTConverter;
import org.apache.hadoop.hive.ql.optimizer.calcite.translator.HiveOpConverter;
@@ -953,16 +958,31 @@ public class CalcitePlanner extends SemanticAnalyzer {
basePlan = hepPlan(basePlan, true, mdProvider, HiveExpandDistinctAggregatesRule.INSTANCE);
}
- // 1. Push Down Semi Joins
+ // 1. Push down limit through outer join
+ if (conf.getBoolVar(HiveConf.ConfVars.HIVE_OPTIMIZE_LIMIT_JOIN_TRANSPOSE)) {
+ // This should be a cost based decision, but till we enable the extended cost
+ // model, we will use the given value for the variable
+ final float reductionProportion = HiveConf.getFloatVar(conf,
+ HiveConf.ConfVars.HIVE_OPTIMIZE_LIMIT_JOIN_TRANSPOSE_REDUCTION_PERCENTAGE);
+ final long reductionTuples = HiveConf.getLongVar(conf,
+ HiveConf.ConfVars.HIVE_OPTIMIZE_LIMIT_JOIN_TRANSPOSE_REDUCTION_TUPLES);
+ basePlan = hepPlan(basePlan, true, mdProvider, HiveSortMergeRule.INSTANCE,
+ HiveSortProjectTransposeRule.INSTANCE, HiveSortJoinReduceRule.INSTANCE);
+ basePlan = hepPlan(basePlan, true, mdProvider, HepMatchOrder.BOTTOM_UP,
+ new HiveSortRemoveRule(reductionProportion, reductionTuples),
+ HiveProjectSortTransposeRule.INSTANCE);
+ }
+
+ // 2. Push Down Semi Joins
basePlan = hepPlan(basePlan, true, mdProvider, SemiJoinJoinTransposeRule.INSTANCE,
SemiJoinFilterTransposeRule.INSTANCE, SemiJoinProjectTransposeRule.INSTANCE);
- // 2. Add not null filters
+ // 3. Add not null filters
if (conf.getBoolVar(HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP)) {
basePlan = hepPlan(basePlan, true, mdProvider, HiveJoinAddNotNullRule.INSTANCE);
}
- // 3. Constant propagation, common filter extraction, and PPD
+ // 4. Constant propagation, common filter extraction, and PPD
basePlan = hepPlan(basePlan, true, mdProvider,
ReduceExpressionsRule.PROJECT_INSTANCE,
ReduceExpressionsRule.FILTER_INSTANCE,
@@ -976,12 +996,12 @@ public class CalcitePlanner extends SemanticAnalyzer {
new FilterAggregateTransposeRule(Filter.class,
HiveFilter.DEFAULT_FILTER_FACTORY, Aggregate.class));
- // 4. Transitive inference & Partition Pruning
+ // 5. Transitive inference & Partition Pruning
basePlan = hepPlan(basePlan, false, mdProvider, new HiveJoinPushTransitivePredicatesRule(
Join.class, HiveFilter.DEFAULT_FILTER_FACTORY),
new HivePartitionPruneRule(conf));
- // 5. Projection Pruning
+ // 6. Projection Pruning
HiveRelFieldTrimmer fieldTrimmer = new HiveRelFieldTrimmer(null,
cluster, HiveProject.DEFAULT_PROJECT_FACTORY,
HiveFilter.DEFAULT_FILTER_FACTORY, HiveJoin.HIVE_JOIN_FACTORY,
@@ -989,7 +1009,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
HiveAggregate.HIVE_AGGR_REL_FACTORY, HiveUnion.UNION_REL_FACTORY);
basePlan = fieldTrimmer.trim(basePlan);
- // 6. Rerun PPD through Project as column pruning would have introduced DT
+ // 7. Rerun PPD through Project as column pruning would have introduced DT
// above scans
basePlan = hepPlan(basePlan, true, mdProvider,
new FilterProjectTransposeRule(Filter.class, HiveFilter.DEFAULT_FILTER_FACTORY,
http://git-wip-us.apache.org/repos/asf/hive/blob/39bd58b2/ql/src/test/queries/clientpositive/limit_join_transpose.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/limit_join_transpose.q b/ql/src/test/queries/clientpositive/limit_join_transpose.q
new file mode 100644
index 0000000..bfc2378
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/limit_join_transpose.q
@@ -0,0 +1,99 @@
+set hive.optimize.limitjointranspose=false;
+
+explain
+select *
+from src src1 left outer join src src2
+on src1.key = src2.key
+limit 1;
+
+select *
+from src src1 left outer join src src2
+on src1.key = src2.key
+limit 1;
+
+
+set hive.optimize.limitjointranspose=true;
+set hive.optimize.limitjointranspose.reductionpercentage=0.0001f;
+set hive.optimize.limitjointranspose.reductiontuples=10;
+
+explain
+select *
+from src src1 left outer join src src2
+on src1.key = src2.key
+limit 1;
+
+select *
+from src src1 left outer join src src2
+on src1.key = src2.key
+limit 1;
+
+
+set hive.optimize.limitjointranspose.reductionpercentage=0.1f;
+set hive.optimize.limitjointranspose.reductiontuples=10;
+
+explain
+select *
+from src src1 left outer join src src2
+on src1.key = src2.key
+limit 1;
+
+select *
+from src src1 left outer join src src2
+on src1.key = src2.key
+limit 1;
+
+explain
+select *
+from src src1 right outer join (
+ select *
+ from src src2 left outer join src src3
+ on src2.value = src3.value) src2
+on src1.key = src2.key
+limit 1;
+
+select *
+from src src1 right outer join (
+ select *
+ from src src2 left outer join src src3
+ on src2.value = src3.value) src2
+on src1.key = src2.key
+limit 1;
+
+set hive.optimize.limitjointranspose.reductionpercentage=1f;
+set hive.optimize.limitjointranspose.reductiontuples=0;
+
+explain
+select *
+from src src1 right outer join (
+ select *
+ from src src2 left outer join src src3
+ on src2.value = src3.value) src2
+on src1.key = src2.key
+limit 1;
+
+select *
+from src src1 right outer join (
+ select *
+ from src src2 left outer join src src3
+ on src2.value = src3.value) src2
+on src1.key = src2.key
+limit 1;
+
+explain
+select *
+from src src1 right outer join (
+ select src2.key, src2.value
+ from src src2 left outer join src src3
+ on src2.value = src3.value) src2
+on src1.key = src2.key
+order by src2.key
+limit 1;
+
+select *
+from src src1 right outer join (
+ select src2.key, src2.value
+ from src src2 left outer join src src3
+ on src2.value = src3.value) src2
+on src1.key = src2.key
+order by src2.key
+limit 1;
http://git-wip-us.apache.org/repos/asf/hive/blob/39bd58b2/ql/src/test/queries/clientpositive/tez_dynpart_hashjoin_3.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/tez_dynpart_hashjoin_3.q b/ql/src/test/queries/clientpositive/tez_dynpart_hashjoin_3.q
new file mode 100644
index 0000000..1994b40
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/tez_dynpart_hashjoin_3.q
@@ -0,0 +1,24 @@
+set hive.optimize.limitjointranspose=true;
+set hive.optimize.limitjointranspose.reductionpercentage=0.1f;
+set hive.optimize.limitjointranspose.reductiontuples=100;
+set hive.explain.user=false;
+set hive.auto.convert.join=false;
+set hive.optimize.dynamic.partition.hashjoin=false;
+
+explain
+select a.*
+from alltypesorc a left outer join src b
+on a.cint = cast(b.key as int) and (a.cint < 100)
+limit 1;
+
+
+set hive.auto.convert.join=true;
+set hive.optimize.dynamic.partition.hashjoin=true;
+set hive.auto.convert.join.noconditionaltask.size=200000;
+set hive.exec.reducers.bytes.per.reducer=200000;
+
+explain
+select a.*
+from alltypesorc a left outer join src b
+on a.cint = cast(b.key as int) and (a.cint < 100)
+limit 1;
http://git-wip-us.apache.org/repos/asf/hive/blob/39bd58b2/ql/src/test/results/clientpositive/join32_lessSize.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/join32_lessSize.q.out b/ql/src/test/results/clientpositive/join32_lessSize.q.out
index 5ea4024..8e71710 100644
--- a/ql/src/test/results/clientpositive/join32_lessSize.q.out
+++ b/ql/src/test/results/clientpositive/join32_lessSize.q.out
@@ -1920,22 +1920,22 @@ TOK_QUERY
STAGE DEPENDENCIES:
- Stage-9 is a root stage
- Stage-7 depends on stages: Stage-9
- Stage-8 depends on stages: Stage-7
+ Stage-8 is a root stage
Stage-6 depends on stages: Stage-8
- Stage-0 depends on stages: Stage-6
- Stage-3 depends on stages: Stage-0
+ Stage-7 depends on stages: Stage-6
+ Stage-5 depends on stages: Stage-7
+ Stage-0 depends on stages: Stage-5
+ Stage-2 depends on stages: Stage-0
STAGE PLANS:
- Stage: Stage-9
+ Stage: Stage-8
Map Reduce Local Work
Alias -> Map Local Tables:
- $hdt$_0:$hdt$_0:y
+ $hdt$_1:$hdt$_1:y
Fetch Operator
limit: -1
Alias -> Map Local Operator Tree:
- $hdt$_0:$hdt$_0:y
+ $hdt$_1:$hdt$_1:y
TableScan
alias: y
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
@@ -1950,7 +1950,7 @@ STAGE PLANS:
1 _col0 (type: string)
Position of Big Table: 1
- Stage: Stage-7
+ Stage: Stage-6
Map Reduce
Map Operator Tree:
TableScan
@@ -2089,12 +2089,12 @@ STAGE PLANS:
name: default.src1
name: default.src1
Truncated Path -> Alias:
- /src1 [$hdt$_0:$hdt$_1:x]
+ /src1 [$hdt$_1:$hdt$_2:x]
- Stage: Stage-8
+ Stage: Stage-7
Map Reduce Local Work
Alias -> Map Local Tables:
- $hdt$_1:z
+ $hdt$_0:z
Fetch Operator
limit: -1
Partition Description:
@@ -2144,7 +2144,7 @@ STAGE PLANS:
name: default.srcpart
name: default.srcpart
Alias -> Map Local Operator Tree:
- $hdt$_1:z
+ $hdt$_0:z
TableScan
alias: z
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
@@ -2159,11 +2159,11 @@ STAGE PLANS:
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
HashTable Sink Operator
keys:
- 0 _col1 (type: string)
- 1 _col0 (type: string)
- Position of Big Table: 0
+ 0 _col0 (type: string)
+ 1 _col1 (type: string)
+ Position of Big Table: 1
- Stage: Stage-6
+ Stage: Stage-5
Map Reduce
Map Operator Tree:
TableScan
@@ -2172,13 +2172,13 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col1 (type: string)
- 1 _col0 (type: string)
- outputColumnNames: _col0, _col1, _col2
- Position of Big Table: 0
+ 0 _col0 (type: string)
+ 1 _col1 (type: string)
+ outputColumnNames: _col0, _col3, _col4
+ Position of Big Table: 1
Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: string), _col2 (type: string), _col1 (type: string)
+ expressions: _col3 (type: string), _col0 (type: string), _col4 (type: string)
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE
File Output Operator
@@ -2313,7 +2313,7 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.dest_j2
- Stage: Stage-3
+ Stage: Stage-2
Stats-Aggr Operator
#### A masked pattern was here ####
http://git-wip-us.apache.org/repos/asf/hive/blob/39bd58b2/ql/src/test/results/clientpositive/join_merging.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/join_merging.q.out b/ql/src/test/results/clientpositive/join_merging.q.out
index b42bdc5..9ab9587 100644
--- a/ql/src/test/results/clientpositive/join_merging.q.out
+++ b/ql/src/test/results/clientpositive/join_merging.q.out
@@ -97,12 +97,12 @@ from part p1 left outer join part p2 on p1.p_partkey = p2.p_partkey
p1.p_size > 10 and p1.p_size > p2.p_size + 10
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-2 depends on stages: Stage-1
- Stage-0 depends on stages: Stage-2
+ Stage-2 is a root stage
+ Stage-1 depends on stages: Stage-2
+ Stage-0 depends on stages: Stage-1
STAGE PLANS:
- Stage: Stage-1
+ Stage: Stage-2
Map Reduce
Map Operator Tree:
TableScan
@@ -153,17 +153,10 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
- Stage: Stage-2
+ Stage: Stage-1
Map Reduce
Map Operator Tree:
TableScan
- Reduce Output Operator
- key expressions: _col2 (type: int)
- sort order: +
- Map-reduce partition columns: _col2 (type: int)
- Statistics: Num rows: 9 Data size: 1112 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: int), _col3 (type: int)
- TableScan
alias: p1
Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
Select Operator
@@ -175,17 +168,24 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
+ TableScan
+ Reduce Output Operator
+ key expressions: _col2 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col2 (type: int)
+ Statistics: Num rows: 9 Data size: 1112 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: int), _col3 (type: int)
Reduce Operator Tree:
Join Operator
condition map:
- Right Outer Join0 to 1
+ Left Outer Join0 to 1
keys:
- 0 _col2 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col1, _col3
+ 0 _col0 (type: int)
+ 1 _col2 (type: int)
+ outputColumnNames: _col2, _col4
Statistics: Num rows: 28 Data size: 3461 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col1 (type: int), _col3 (type: int)
+ expressions: _col2 (type: int), _col4 (type: int)
outputColumnNames: _col0, _col1
Statistics: Num rows: 28 Data size: 3461 Basic stats: COMPLETE Column stats: NONE
File Output Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/39bd58b2/ql/src/test/results/clientpositive/limit_join_transpose.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/limit_join_transpose.q.out b/ql/src/test/results/clientpositive/limit_join_transpose.q.out
new file mode 100644
index 0000000..8717154
--- /dev/null
+++ b/ql/src/test/results/clientpositive/limit_join_transpose.q.out
@@ -0,0 +1,869 @@
+PREHOOK: query: explain
+select *
+from src src1 left outer join src src2
+on src1.key = src2.key
+limit 1
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select *
+from src src1 left outer join src src2
+on src1.key = src2.key
+limit 1
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: src1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ TableScan
+ alias: src1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 1
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select *
+from src src1 left outer join src src2
+on src1.key = src2.key
+limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select *
+from src src1 left outer join src src2
+on src1.key = src2.key
+limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+0 val_0 0 val_0
+PREHOOK: query: explain
+select *
+from src src1 left outer join src src2
+on src1.key = src2.key
+limit 1
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select *
+from src src1 left outer join src src2
+on src1.key = src2.key
+limit 1
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: src1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ TableScan
+ alias: src1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 1
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select *
+from src src1 left outer join src src2
+on src1.key = src2.key
+limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select *
+from src src1 left outer join src src2
+on src1.key = src2.key
+limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+0 val_0 0 val_0
+PREHOOK: query: explain
+select *
+from src src1 left outer join src src2
+on src1.key = src2.key
+limit 1
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select *
+from src src1 left outer join src src2
+on src1.key = src2.key
+limit 1
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: src1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 1
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string), _col1 (type: string)
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: string), VALUE._col1 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 1
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ TableScan
+ alias: src1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 1
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select *
+from src src1 left outer join src src2
+on src1.key = src2.key
+limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select *
+from src src1 left outer join src src2
+on src1.key = src2.key
+limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+238 val_238 238 val_238
+PREHOOK: query: explain
+select *
+from src src1 right outer join (
+ select *
+ from src src2 left outer join src src3
+ on src2.value = src3.value) src2
+on src1.key = src2.key
+limit 1
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select *
+from src src1 right outer join (
+ select *
+ from src src2 left outer join src src3
+ on src2.value = src3.value) src2
+on src1.key = src2.key
+limit 1
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-3 depends on stages: Stage-2
+ Stage-0 depends on stages: Stage-3
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: src1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 1
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string), _col1 (type: string)
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: string), VALUE._col1 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 1
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: string)
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
+ TableScan
+ alias: src1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ keys:
+ 0 _col1 (type: string)
+ 1 _col1 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-3
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col2 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col2 (type: string)
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string)
+ TableScan
+ alias: src1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ keys:
+ 0 _col2 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col4 (type: string), _col5 (type: string), _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 1
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select *
+from src src1 right outer join (
+ select *
+ from src src2 left outer join src src3
+ on src2.value = src3.value) src2
+on src1.key = src2.key
+limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select *
+from src src1 right outer join (
+ select *
+ from src src2 left outer join src src3
+ on src2.value = src3.value) src2
+on src1.key = src2.key
+limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+238 val_238 238 val_238 238 val_238
+PREHOOK: query: explain
+select *
+from src src1 right outer join (
+ select *
+ from src src2 left outer join src src3
+ on src2.value = src3.value) src2
+on src1.key = src2.key
+limit 1
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select *
+from src src1 right outer join (
+ select *
+ from src src2 left outer join src src3
+ on src2.value = src3.value) src2
+on src1.key = src2.key
+limit 1
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-2 is a root stage
+ Stage-3 depends on stages: Stage-2
+ Stage-4 depends on stages: Stage-3
+ Stage-1 depends on stages: Stage-4
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: src1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 1
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string), _col1 (type: string)
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: string), VALUE._col1 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 1
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-3
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: string)
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
+ TableScan
+ alias: src1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ keys:
+ 0 _col1 (type: string)
+ 1 _col1 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 1
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-4
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string)
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 1
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: src1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ TableScan
+ Reduce Output Operator
+ key expressions: _col2 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col2 (type: string)
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string)
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Right Outer Join0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col2 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 1
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select *
+from src src1 right outer join (
+ select *
+ from src src2 left outer join src src3
+ on src2.value = src3.value) src2
+on src1.key = src2.key
+limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select *
+from src src1 right outer join (
+ select *
+ from src src2 left outer join src src3
+ on src2.value = src3.value) src2
+on src1.key = src2.key
+limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+238 val_238 238 val_238 238 val_238
+PREHOOK: query: explain
+select *
+from src src1 right outer join (
+ select src2.key, src2.value
+ from src src2 left outer join src src3
+ on src2.value = src3.value) src2
+on src1.key = src2.key
+order by src2.key
+limit 1
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select *
+from src src1 right outer join (
+ select src2.key, src2.value
+ from src src2 left outer join src src3
+ on src2.value = src3.value) src2
+on src1.key = src2.key
+order by src2.key
+limit 1
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-5 is a root stage
+ Stage-3 depends on stages: Stage-5
+ Stage-4 depends on stages: Stage-3
+ Stage-1 depends on stages: Stage-4
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+
+STAGE PLANS:
+ Stage: Stage-5
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: src1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 1
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-3
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: src1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: value (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ TableScan
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: string)
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Right Outer Join0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col1 (type: string)
+ outputColumnNames: _col1, _col2
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col1 (type: string), _col2 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-4
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 1
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: src1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Right Outer Join0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col2 (type: string)
+ sort order: +
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string)
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col2 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 1
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select *
+from src src1 right outer join (
+ select src2.key, src2.value
+ from src src2 left outer join src src3
+ on src2.value = src3.value) src2
+on src1.key = src2.key
+order by src2.key
+limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select *
+from src src1 right outer join (
+ select src2.key, src2.value
+ from src src2 left outer join src src3
+ on src2.value = src3.value) src2
+on src1.key = src2.key
+order by src2.key
+limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+0 val_0 0 val_0
[11/12] hive git commit: HIVE-12319 : Remove
HadoopShims::getHadoopConfNames() (Aleksei Statkevich via Ashutosh Chauhan)
Posted by om...@apache.org.
HIVE-12319 : Remove HadoopShims::getHadoopConfNames() (Aleksei Statkevich via Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b9650be8
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b9650be8
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b9650be8
Branch: refs/heads/master-fixed
Commit: b9650be8e48da18b357d502c4d4be2700d956469
Parents: 82705f0
Author: Aleksei Statkevich <me...@gmail.com>
Authored: Sun Nov 15 20:55:00 2015 -0800
Committer: Owen O'Malley <om...@apache.org>
Committed: Fri Nov 20 13:46:23 2015 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 21 ++++++-----------
.../metrics/metrics2/TestCodahaleMetrics.java | 6 ++---
.../rcfile/RCFileMapReduceInputFormat.java | 8 +++----
.../rcfile/TestRCFileMapReduceInputFormat.java | 4 ++--
...estDDLWithRemoteMetastoreSecondNamenode.java | 3 ++-
.../org/apache/hadoop/hive/ql/QTestUtil.java | 3 ++-
.../java/org/apache/hadoop/hive/ql/Driver.java | 7 +++---
.../org/apache/hadoop/hive/ql/exec/DDLTask.java | 5 ++--
.../hadoop/hive/ql/exec/FetchOperator.java | 3 ++-
.../apache/hadoop/hive/ql/exec/Utilities.java | 6 ++---
.../hadoop/hive/ql/exec/mr/ExecDriver.java | 8 +++----
.../hive/ql/exec/persistence/RowContainer.java | 8 ++++---
.../hadoop/hive/ql/exec/tez/DagUtils.java | 2 +-
.../hive/ql/exec/tez/HiveSplitGenerator.java | 15 ++----------
.../hive/ql/io/CombineHiveInputFormat.java | 7 +++---
.../hadoop/hive/ql/io/HiveFileFormatUtils.java | 6 ++---
.../hadoop/hive/ql/io/HiveInputFormat.java | 5 +---
.../hadoop/hive/ql/io/merge/MergeFileTask.java | 6 ++---
.../hadoop/hive/ql/io/orc/OrcInputFormat.java | 8 ++-----
.../ql/io/rcfile/stats/PartialScanTask.java | 6 ++---
.../rcfile/truncate/ColumnTruncateMapper.java | 4 ++--
.../io/rcfile/truncate/ColumnTruncateTask.java | 7 +++---
.../hadoop/hive/ql/exec/TestOperators.java | 5 ++--
.../apache/hadoop/hive/ql/io/TestRCFile.java | 9 ++------
.../hive/ql/io/orc/TestInputOutputFormat.java | 20 ++++++++--------
.../hive/ql/io/orc/TestOrcSplitElimination.java | 14 ++++++------
.../apache/hadoop/hive/shims/Hadoop23Shims.java | 24 --------------------
.../apache/hadoop/hive/shims/HadoopShims.java | 3 ---
.../hadoop/hive/shims/HadoopShimsSecure.java | 15 ++++++++----
29 files changed, 95 insertions(+), 143 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b9650be8/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 0afb964..4f8209a 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -28,9 +28,10 @@ import org.apache.hadoop.hive.conf.Validator.RangeValidator;
import org.apache.hadoop.hive.conf.Validator.RatioValidator;
import org.apache.hadoop.hive.conf.Validator.StringSet;
import org.apache.hadoop.hive.conf.Validator.TimeValidator;
-import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.hive.shims.Utils;
import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapreduce.lib.input.CombineFileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Shell;
import org.apache.hive.common.HiveCompat;
@@ -386,22 +387,14 @@ public class HiveConf extends Configuration {
HADOOPBIN("hadoop.bin.path", findHadoopBinary(), "", true),
HIVE_FS_HAR_IMPL("fs.har.impl", "org.apache.hadoop.hive.shims.HiveHarFileSystem",
"The implementation for accessing Hadoop Archives. Note that this won't be applicable to Hadoop versions less than 0.20"),
- HADOOPFS(ShimLoader.getHadoopShims().getHadoopConfNames().get("HADOOPFS"), null, "", true),
- HADOOPMAPFILENAME(ShimLoader.getHadoopShims().getHadoopConfNames().get("HADOOPMAPFILENAME"), null, "", true),
- HADOOPMAPREDINPUTDIR(ShimLoader.getHadoopShims().getHadoopConfNames().get("HADOOPMAPREDINPUTDIR"), null, "", true),
- HADOOPMAPREDINPUTDIRRECURSIVE(ShimLoader.getHadoopShims().getHadoopConfNames().get("HADOOPMAPREDINPUTDIRRECURSIVE"), false, "", true),
- MAPREDMAXSPLITSIZE(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMAXSPLITSIZE"), 256000000L, "", true),
- MAPREDMINSPLITSIZE(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZE"), 1L, "", true),
- MAPREDMINSPLITSIZEPERNODE(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZEPERNODE"), 1L, "", true),
- MAPREDMINSPLITSIZEPERRACK(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZEPERRACK"), 1L, "", true),
+ MAPREDMAXSPLITSIZE(FileInputFormat.SPLIT_MAXSIZE, 256000000L, "", true),
+ MAPREDMINSPLITSIZE(FileInputFormat.SPLIT_MINSIZE, 1L, "", true),
+ MAPREDMINSPLITSIZEPERNODE(CombineFileInputFormat.SPLIT_MINSIZE_PERNODE, 1L, "", true),
+ MAPREDMINSPLITSIZEPERRACK(CombineFileInputFormat.SPLIT_MINSIZE_PERRACK, 1L, "", true),
// The number of reduce tasks per job. Hadoop sets this value to 1 by default
// By setting this property to -1, Hive will automatically determine the correct
// number of reducers.
- HADOOPNUMREDUCERS(ShimLoader.getHadoopShims().getHadoopConfNames().get("HADOOPNUMREDUCERS"), -1, "", true),
- HADOOPJOBNAME(ShimLoader.getHadoopShims().getHadoopConfNames().get("HADOOPJOBNAME"), null, "", true),
- HADOOPSPECULATIVEEXECREDUCERS(ShimLoader.getHadoopShims().getHadoopConfNames().get("HADOOPSPECULATIVEEXECREDUCERS"), true, "", true),
- MAPREDSETUPCLEANUPNEEDED(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDSETUPCLEANUPNEEDED"), false, "", true),
- MAPREDTASKCLEANUPNEEDED(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDTASKCLEANUPNEEDED"), false, "", true),
+ HADOOPNUMREDUCERS("mapreduce.job.reduces", -1, "", true),
// Metastore stuff. Be sure to update HiveConf.metaVars when you add something here!
METASTOREWAREHOUSE("hive.metastore.warehouse.dir", "/user/hive/warehouse",
http://git-wip-us.apache.org/repos/asf/hive/blob/b9650be8/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java
----------------------------------------------------------------------
diff --git a/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java b/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java
index 27825b1..55bb2c2 100644
--- a/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java
+++ b/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java
@@ -22,11 +22,10 @@ import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Timer;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.hadoop.hive.common.metrics.common.MetricsConstant;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.hive.common.metrics.common.MetricsFactory;
import org.apache.hadoop.hive.common.metrics.common.MetricsVariable;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.shims.ShimLoader;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -57,8 +56,7 @@ public class TestCodahaleMetrics {
jsonReportFile = new File(workDir, "json_reporting");
jsonReportFile.delete();
- String defaultFsName = ShimLoader.getHadoopShims().getHadoopConfNames().get("HADOOPFS");
- conf.set(defaultFsName, "local");
+ conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, "local");
conf.setVar(HiveConf.ConfVars.HIVE_METRICS_CLASS, CodahaleMetrics.class.getCanonicalName());
conf.setVar(HiveConf.ConfVars.HIVE_METRICS_REPORTER, MetricsReporting.JSON_FILE.name() + "," + MetricsReporting.JMX.name());
conf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_LOCATION, jsonReportFile.toString());
http://git-wip-us.apache.org/repos/asf/hive/blob/b9650be8/hcatalog/core/src/main/java/org/apache/hive/hcatalog/rcfile/RCFileMapReduceInputFormat.java
----------------------------------------------------------------------
diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/rcfile/RCFileMapReduceInputFormat.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/rcfile/RCFileMapReduceInputFormat.java
index 0c3d707..d5fa93c 100644
--- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/rcfile/RCFileMapReduceInputFormat.java
+++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/rcfile/RCFileMapReduceInputFormat.java
@@ -21,8 +21,8 @@ package org.apache.hive.hcatalog.rcfile;
import java.io.IOException;
import java.util.List;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.serde2.columnar.BytesRefArrayWritable;
-import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.mapreduce.InputSplit;
@@ -44,10 +44,8 @@ public class RCFileMapReduceInputFormat<K extends LongWritable, V extends BytesR
@Override
public List<InputSplit> getSplits(JobContext job) throws IOException {
-
- job.getConfiguration().setLong(
- ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZE"),
- SequenceFile.SYNC_INTERVAL);
+ HiveConf.setLongVar(job.getConfiguration(),
+ HiveConf.ConfVars.MAPREDMINSPLITSIZE, SequenceFile.SYNC_INTERVAL);
return super.getSplits(job);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/b9650be8/hcatalog/core/src/test/java/org/apache/hive/hcatalog/rcfile/TestRCFileMapReduceInputFormat.java
----------------------------------------------------------------------
diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/rcfile/TestRCFileMapReduceInputFormat.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/rcfile/TestRCFileMapReduceInputFormat.java
index 02a13b3..bb6c582 100644
--- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/rcfile/TestRCFileMapReduceInputFormat.java
+++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/rcfile/TestRCFileMapReduceInputFormat.java
@@ -230,8 +230,8 @@ public class TestRCFileMapReduceInputFormat extends TestCase {
Configuration jonconf = new Configuration(cloneConf);
jonconf.set("mapred.input.dir", testDir.toString());
JobContext context = new Job(jonconf);
- context.getConfiguration().setLong(
- ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMAXSPLITSIZE"), maxSplitSize);
+ HiveConf.setLongVar(context.getConfiguration(),
+ HiveConf.ConfVars.MAPREDMAXSPLITSIZE, maxSplitSize);
List<InputSplit> splits = inputFormat.getSplits(context);
assertEquals("splits length should be " + splitNumber, splits.size(), splitNumber);
int readCount = 0;
http://git-wip-us.apache.org/repos/asf/hive/blob/b9650be8/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java
index 7ed6436..bfb25aa 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java
@@ -23,6 +23,7 @@ import java.util.HashMap;
import junit.framework.JUnit4TestAdapter;
import junit.framework.TestCase;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
@@ -97,7 +98,7 @@ public class TestDDLWithRemoteMetastoreSecondNamenode extends TestCase {
}
fs2.mkdirs(tmppathFs2);
fs2Uri = fs2.getUri().toString();
- jobConf.setVar(HiveConf.ConfVars.HADOOPFS, fs2Uri);
+ jobConf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, fs2Uri);
driver = new Driver(jobConf);
http://git-wip-us.apache.org/repos/asf/hive/blob/b9650be8/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index bf6bea7..ac1f15e 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -56,6 +56,7 @@ import java.util.regex.Pattern;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@@ -299,7 +300,7 @@ public class QTestUtil {
// set fs.default.name to the uri of mini-dfs
String dfsUriString = WindowsPathUtil.getHdfsUriString(dfs.getFileSystem().getUri().toString());
- conf.setVar(HiveConf.ConfVars.HADOOPFS, dfsUriString);
+ conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, dfsUriString);
// hive.metastore.warehouse.dir needs to be set relative to the mini-dfs
conf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE,
(new Path(dfsUriString,
http://git-wip-us.apache.org/repos/asf/hive/blob/b9650be8/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index fbde9eb..8fafd61 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -36,6 +36,7 @@ import java.util.Set;
import java.util.concurrent.locks.ReentrantLock;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.mapreduce.MRJobConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.fs.FSDataInputStream;
@@ -1459,7 +1460,7 @@ public class Driver implements CommandProcessor {
public int execute() throws CommandNeedRetryException {
PerfLogger perfLogger = SessionState.getPerfLogger();
perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.DRIVER_EXECUTE);
- boolean noName = StringUtils.isEmpty(conf.getVar(HiveConf.ConfVars.HADOOPJOBNAME));
+ boolean noName = StringUtils.isEmpty(conf.get(MRJobConfig.JOB_NAME));
int maxlen = conf.getIntVar(HiveConf.ConfVars.HIVEJOBNAMELENGTH);
String queryId = plan.getQueryId();
@@ -1703,7 +1704,7 @@ public class Driver implements CommandProcessor {
SessionState.get().getHiveHistory().endQuery(queryId);
}
if (noName) {
- conf.setVar(HiveConf.ConfVars.HADOOPJOBNAME, "");
+ conf.set(MRJobConfig.JOB_NAME, "");
}
dumpMetaCallTimingWithoutEx("execution");
perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.DRIVER_EXECUTE);
@@ -1786,7 +1787,7 @@ public class Driver implements CommandProcessor {
}
if (tsk.isMapRedTask() && !(tsk instanceof ConditionalTask)) {
if (noName) {
- conf.setVar(HiveConf.ConfVars.HADOOPJOBNAME, jobname + "(" + tsk.getId() + ")");
+ conf.set(MRJobConfig.JOB_NAME, jobname + "(" + tsk.getId() + ")");
}
conf.set("mapreduce.workflow.node.name", tsk.getId());
Utilities.setWorkflowAdjacencies(conf, plan);
http://git-wip-us.apache.org/repos/asf/hive/blob/b9650be8/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index 9ab3e98..a210b95 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hive.ql.exec;
import com.google.common.collect.Iterables;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.mapreduce.MRJobConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -1368,7 +1369,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
String jobname = String.format("Archiving %s@%s",
tbl.getTableName(), partSpecInfo.getName());
jobname = Utilities.abbreviate(jobname, maxJobNameLen - 6);
- conf.setVar(HiveConf.ConfVars.HADOOPJOBNAME, jobname);
+ conf.set(MRJobConfig.JOB_NAME, jobname);
HadoopArchives har = new HadoopArchives(conf);
List<String> args = new ArrayList<String>();
@@ -1378,7 +1379,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
args.add(originalDir.toString());
args.add(tmpPath.toString());
- ret = ToolRunner.run(har, args.toArray(new String[0]));;
+ ret = ToolRunner.run(har, args.toArray(new String[0]));
} catch (Exception e) {
throw new HiveException(e);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/b9650be8/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
index ad36093..3aafe89 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
@@ -66,6 +66,7 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.JobConfigurable;
import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.util.StringUtils;
import org.apache.hive.common.util.AnnotationUtils;
import org.apache.hive.common.util.ReflectionUtil;
@@ -648,7 +649,7 @@ public class FetchOperator implements Serializable {
* @return list of file status entries
*/
private FileStatus[] listStatusUnderPath(FileSystem fs, Path p) throws IOException {
- boolean recursive = HiveConf.getBoolVar(job, HiveConf.ConfVars.HADOOPMAPREDINPUTDIRRECURSIVE);
+ boolean recursive = job.getBoolean(FileInputFormat.INPUT_DIR_RECURSIVE, false);
// If this is in acid format always read it recursively regardless of what the jobconf says.
if (!recursive && !AcidUtils.isAcid(p, job)) {
return fs.listStatus(p, FileUtils.HIDDEN_FILES_PATH_FILTER);
http://git-wip-us.apache.org/repos/asf/hive/blob/b9650be8/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index 0700e2f..9dbb45a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -75,7 +75,6 @@ import java.util.concurrent.Future;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.zip.Deflater;
@@ -92,6 +91,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.filecache.DistributedCache;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -207,7 +207,6 @@ import org.apache.hadoop.mapred.SequenceFileOutputFormat;
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.Shell;
import org.apache.hive.common.util.ReflectionUtil;
-import org.slf4j.Logger;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.io.Input;
@@ -3952,7 +3951,8 @@ public final class Utilities {
* @return True/False
*/
public static boolean isDefaultNameNode(HiveConf conf) {
- return !conf.getChangedProperties().containsKey(HiveConf.ConfVars.HADOOPFS.varname);
+ return !conf.getChangedProperties().containsKey(
+ CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY);
}
/**
http://git-wip-us.apache.org/repos/asf/hive/blob/b9650be8/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
index 380cf08..df96d8c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
@@ -31,6 +31,7 @@ import java.util.List;
import java.util.Properties;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.mapreduce.MRJobConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@@ -275,8 +276,7 @@ public class ExecDriver extends Task<MapredWork> implements Serializable, Hadoop
// Turn on speculative execution for reducers
boolean useSpeculativeExecReducers = HiveConf.getBoolVar(job,
HiveConf.ConfVars.HIVESPECULATIVEEXECREDUCERS);
- HiveConf.setBoolVar(job, HiveConf.ConfVars.HADOOPSPECULATIVEEXECREDUCERS,
- useSpeculativeExecReducers);
+ job.setBoolean(MRJobConfig.REDUCE_SPECULATIVE, useSpeculativeExecReducers);
String inpFormat = HiveConf.getVar(job, HiveConf.ConfVars.HIVEINPUTFORMAT);
@@ -316,11 +316,11 @@ public class ExecDriver extends Task<MapredWork> implements Serializable, Hadoop
initializeFiles("tmpfiles", addedFiles);
}
int returnVal = 0;
- boolean noName = StringUtils.isEmpty(HiveConf.getVar(job, HiveConf.ConfVars.HADOOPJOBNAME));
+ boolean noName = StringUtils.isEmpty(job.get(MRJobConfig.JOB_NAME));
if (noName) {
// This is for a special case to ensure unit tests pass
- HiveConf.setVar(job, HiveConf.ConfVars.HADOOPJOBNAME, "JOB" + Utilities.randGen.nextInt());
+ job.set(MRJobConfig.JOB_NAME, "JOB" + Utilities.randGen.nextInt());
}
String addedArchives = HiveConf.getVar(job, HiveConf.ConfVars.HIVEADDEDARCHIVES);
// Transfer HIVEADDEDARCHIVES to "tmparchives" so hadoop understands it
http://git-wip-us.apache.org/repos/asf/hive/blob/b9650be8/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java
index c2d0d68..358f692 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java
@@ -23,13 +23,14 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
@@ -147,7 +148,8 @@ public class RowContainer<ROW extends List<Object>>
private JobConf getLocalFSJobConfClone(Configuration jc) {
if (this.jobCloneUsingLocalFs == null) {
this.jobCloneUsingLocalFs = new JobConf(jc);
- HiveConf.setVar(jobCloneUsingLocalFs, HiveConf.ConfVars.HADOOPFS, Utilities.HADOOP_LOCAL_FS);
+ jobCloneUsingLocalFs.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
+ Utilities.HADOOP_LOCAL_FS);
}
return this.jobCloneUsingLocalFs;
}
@@ -217,7 +219,7 @@ public class RowContainer<ROW extends List<Object>>
tblDesc.getInputFileFormatClass(), localJc);
}
- HiveConf.setVar(localJc, HiveConf.ConfVars.HADOOPMAPREDINPUTDIR,
+ localJc.set(FileInputFormat.INPUT_DIR,
org.apache.hadoop.util.StringUtils.escapeString(parentFile.getAbsolutePath()));
inputSplits = inputFormat.getSplits(localJc, 1);
actualSplitNum = inputSplits.length;
http://git-wip-us.apache.org/repos/asf/hive/blob/b9650be8/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
index 6dcfe8d..db4d73d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
@@ -665,7 +665,7 @@ public class DagUtils {
boolean useSpeculativeExecReducers = HiveConf.getBoolVar(conf,
HiveConf.ConfVars.HIVESPECULATIVEEXECREDUCERS);
- HiveConf.setBoolVar(conf, HiveConf.ConfVars.HADOOPSPECULATIVEEXECREDUCERS,
+ conf.setBoolean(org.apache.hadoop.mapreduce.MRJobConfig.REDUCE_SPECULATIVE,
useSpeculativeExecReducers);
return conf;
http://git-wip-us.apache.org/repos/asf/hive/blob/b9650be8/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java
index 8902acb..532d242 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java
@@ -38,7 +38,6 @@ import org.apache.hadoop.hive.ql.plan.MapWork;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.mapred.FileSplit;
-import org.apache.hadoop.hive.shims.HadoopShims;
import org.apache.hadoop.mapred.InputFormat;
import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.JobConf;
@@ -81,16 +80,6 @@ public class HiveSplitGenerator extends InputInitializer {
private final MapWork work;
private final SplitGrouper splitGrouper = new SplitGrouper();
- private static final String MIN_SPLIT_SIZE;
- @SuppressWarnings("unused")
- private static final String MAX_SPLIT_SIZE;
-
- static {
- final HadoopShims SHIMS = ShimLoader.getHadoopShims();
- MIN_SPLIT_SIZE = SHIMS.getHadoopConfNames().get("MAPREDMINSPLITSIZE");
- MAX_SPLIT_SIZE = SHIMS.getHadoopConfNames().get("MAPREDMAXSPLITSIZE");
- }
-
public HiveSplitGenerator(InputInitializerContext initializerContext) throws IOException,
SerDeException {
super(initializerContext);
@@ -143,7 +132,7 @@ public class HiveSplitGenerator extends InputInitializer {
int taskResource = getContext().getVertexTaskResource().getMemory();
int availableSlots = totalResource / taskResource;
- if (conf.getLong(MIN_SPLIT_SIZE, 1) <= 1) {
+ if (HiveConf.getLongVar(conf, HiveConf.ConfVars.MAPREDMINSPLITSIZE, 1) <= 1) {
// broken configuration from mapred-default.xml
final long blockSize = conf.getLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY,
DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT);
@@ -151,7 +140,7 @@ public class HiveSplitGenerator extends InputInitializer {
TezMapReduceSplitsGrouper.TEZ_GROUPING_SPLIT_MIN_SIZE,
TezMapReduceSplitsGrouper.TEZ_GROUPING_SPLIT_MIN_SIZE_DEFAULT);
final long preferredSplitSize = Math.min(blockSize / 2, minGrouping);
- jobConf.setLong(MIN_SPLIT_SIZE, preferredSplitSize);
+ HiveConf.setLongVar(conf, HiveConf.ConfVars.MAPREDMINSPLITSIZE, preferredSplitSize);
LOG.info("The preferred split size is " + preferredSplitSize);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/b9650be8/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
index 323ac43..a598ccc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
@@ -39,7 +39,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
-import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.log.PerfLogger;
@@ -507,11 +506,11 @@ public class CombineHiveInputFormat<K extends WritableComparable, V extends Writ
}
// Store the previous value for the path specification
- String oldPaths = job.get(HiveConf.ConfVars.HADOOPMAPREDINPUTDIR.varname);
+ String oldPaths = job.get(org.apache.hadoop.mapreduce.lib.input.FileInputFormat.INPUT_DIR);
if (LOG.isDebugEnabled()) {
LOG.debug("The received input paths are: [" + oldPaths +
"] against the property "
- + HiveConf.ConfVars.HADOOPMAPREDINPUTDIR.varname);
+ + org.apache.hadoop.mapreduce.lib.input.FileInputFormat.INPUT_DIR);
}
// Process the normal splits
@@ -540,7 +539,7 @@ public class CombineHiveInputFormat<K extends WritableComparable, V extends Writ
// This is just to prevent incompatibilities with previous versions Hive
// if some application depends on the original value being set.
if (oldPaths != null) {
- job.set(HiveConf.ConfVars.HADOOPMAPREDINPUTDIR.varname, oldPaths);
+ job.set(org.apache.hadoop.mapreduce.lib.input.FileInputFormat.INPUT_DIR, oldPaths);
}
// clear work from ThreadLocal after splits generated in case of thread is reused in pool.
http://git-wip-us.apache.org/repos/asf/hive/blob/b9650be8/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java
index bc13862..8b229af 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java
@@ -48,7 +48,6 @@ import org.apache.hadoop.hive.ql.plan.OperatorDesc;
import org.apache.hadoop.hive.ql.plan.PartitionDesc;
import org.apache.hadoop.hive.ql.plan.TableDesc;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.io.SequenceFile.CompressionType;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.compress.CompressionCodec;
@@ -63,6 +62,7 @@ import org.apache.hadoop.mapred.SequenceFileInputFormat;
import org.apache.hadoop.mapred.SequenceFileOutputFormat;
import org.apache.hadoop.mapred.TaskAttemptContext;
import org.apache.hadoop.mapred.TextInputFormat;
+import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.util.Shell;
import org.apache.hive.common.util.ReflectionUtil;
import org.slf4j.Logger;
@@ -567,10 +567,10 @@ public final class HiveFileFormatUtils {
// option to bypass job setup and cleanup was introduced in hadoop-21 (MAPREDUCE-463)
// but can be backported. So we disable setup/cleanup in all versions >= 0.19
- conf.setBoolean(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDSETUPCLEANUPNEEDED"), false);
+ conf.setBoolean(MRJobConfig.SETUP_CLEANUP_NEEDED, false);
// option to bypass task cleanup task was introduced in hadoop-23 (MAPREDUCE-2206)
// but can be backported. So we disable setup/cleanup in all versions >= 0.19
- conf.setBoolean(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDTASKCLEANUPNEEDED"), false);
+ conf.setBoolean(MRJobConfig.TASK_CLEANUP_NEEDED, false);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/b9650be8/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
index b63deb2..3feab1a 100755
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
@@ -54,7 +54,6 @@ import org.apache.hadoop.hive.ql.plan.PartitionDesc;
import org.apache.hadoop.hive.ql.plan.TableScanDesc;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
-import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapred.FileInputFormat;
@@ -360,9 +359,7 @@ public class HiveInputFormat<K extends WritableComparable, V extends Writable>
footerCount = Utilities.getFooterCount(table, conf);
if (headerCount != 0 || footerCount != 0) {
// Input file has header or footer, cannot be splitted.
- conf.setLong(
- ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZE"),
- Long.MAX_VALUE);
+ HiveConf.setLongVar(conf, ConfVars.MAPREDMINSPLITSIZE, Long.MAX_VALUE);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/b9650be8/ql/src/java/org/apache/hadoop/hive/ql/io/merge/MergeFileTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/merge/MergeFileTask.java b/ql/src/java/org/apache/hadoop/hive/ql/io/merge/MergeFileTask.java
index 2f09014..7453145 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/merge/MergeFileTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/merge/MergeFileTask.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RunningJob;
+import org.apache.hadoop.mapreduce.MRJobConfig;
import java.io.IOException;
import java.io.Serializable;
@@ -106,8 +107,7 @@ public class MergeFileTask extends Task<MergeFileWork> implements Serializable,
}
// set job name
- boolean noName = StringUtils.isEmpty(HiveConf.getVar(job,
- HiveConf.ConfVars.HADOOPJOBNAME));
+ boolean noName = StringUtils.isEmpty(job.get(MRJobConfig.JOB_NAME));
String jobName = null;
if (noName && this.getQueryPlan() != null) {
@@ -118,7 +118,7 @@ public class MergeFileTask extends Task<MergeFileWork> implements Serializable,
if (noName) {
// This is for a special case to ensure unit tests pass
- HiveConf.setVar(job, HiveConf.ConfVars.HADOOPJOBNAME,
+ job.set(MRJobConfig.JOB_NAME,
jobName != null ? jobName : "JOB" + Utilities.randGen.nextInt());
}
http://git-wip-us.apache.org/repos/asf/hive/blob/b9650be8/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
index 714af23..0567cbe 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
@@ -129,10 +129,6 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
private static final Logger LOG = LoggerFactory.getLogger(OrcInputFormat.class);
private static boolean isDebugEnabled = LOG.isDebugEnabled();
static final HadoopShims SHIMS = ShimLoader.getHadoopShims();
- static final String MIN_SPLIT_SIZE =
- SHIMS.getHadoopConfNames().get("MAPREDMINSPLITSIZE");
- static final String MAX_SPLIT_SIZE =
- SHIMS.getHadoopConfNames().get("MAPREDMAXSPLITSIZE");
private static final long DEFAULT_MIN_SPLIT_SIZE = 16 * 1024 * 1024;
private static final long DEFAULT_MAX_SPLIT_SIZE = 256 * 1024 * 1024;
@@ -497,8 +493,8 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
Context(Configuration conf, final int minSplits) {
this.conf = conf;
this.sarg = ConvertAstToSearchArg.createFromConf(conf);
- minSize = conf.getLong(MIN_SPLIT_SIZE, DEFAULT_MIN_SPLIT_SIZE);
- maxSize = conf.getLong(MAX_SPLIT_SIZE, DEFAULT_MAX_SPLIT_SIZE);
+ minSize = HiveConf.getLongVar(conf, ConfVars.MAPREDMINSPLITSIZE, DEFAULT_MIN_SPLIT_SIZE);
+ maxSize = HiveConf.getLongVar(conf, ConfVars.MAPREDMAXSPLITSIZE, DEFAULT_MAX_SPLIT_SIZE);
String ss = conf.get(ConfVars.HIVE_ORC_SPLIT_STRATEGY.varname);
if (ss == null || ss.equals(SplitStrategyKind.HYBRID.name())) {
splitStrategyKind = SplitStrategyKind.HYBRID;
http://git-wip-us.apache.org/repos/asf/hive/blob/b9650be8/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java b/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java
index fd04fb5..188e9a6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java
@@ -24,6 +24,7 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.mapreduce.MRJobConfig;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -157,8 +158,7 @@ public class PartialScanTask extends Task<PartialScanWork> implements
int returnVal = 0;
RunningJob rj = null;
- boolean noName = StringUtils.isEmpty(HiveConf.getVar(job,
- HiveConf.ConfVars.HADOOPJOBNAME));
+ boolean noName = StringUtils.isEmpty(job.get(MRJobConfig.JOB_NAME));
String jobName = null;
if (noName && this.getQueryPlan() != null) {
@@ -169,7 +169,7 @@ public class PartialScanTask extends Task<PartialScanWork> implements
if (noName) {
// This is for a special case to ensure unit tests pass
- HiveConf.setVar(job, HiveConf.ConfVars.HADOOPJOBNAME,
+ job.set(MRJobConfig.JOB_NAME,
jobName != null ? jobName : "JOB" + Utilities.randGen.nextInt());
}
http://git-wip-us.apache.org/repos/asf/hive/blob/b9650be8/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateMapper.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateMapper.java b/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateMapper.java
index 34a18cb..bd537cd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateMapper.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateMapper.java
@@ -20,12 +20,12 @@ package org.apache.hadoop.hive.ql.io.rcfile.truncate;
import java.io.IOException;
+import org.apache.hadoop.mapreduce.MRJobConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.io.RCFile;
import org.apache.hadoop.hive.ql.io.RCFileOutputFormat;
@@ -95,7 +95,7 @@ public class ColumnTruncateMapper extends MapReduceBase implements
String taskId = Utilities.getTaskId(jc);
this.tmpPath = tmpPath;
this.taskTmpPath = taskTmpPath;
- String inputFile = HiveConf.getVar(jc, HiveConf.ConfVars.HADOOPMAPFILENAME);
+ String inputFile = jc.get(MRJobConfig.MAP_INPUT_FILE);
int lastSeparator = inputFile.lastIndexOf(Path.SEPARATOR) + 1;
finalPath = new Path(tmpPath, inputFile.substring(lastSeparator));
outPath = new Path(taskTmpPath, Utilities.toTempPath(taskId));
http://git-wip-us.apache.org/repos/asf/hive/blob/b9650be8/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java b/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java
index 79b3cfa..08e3d80 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RunningJob;
+import org.apache.hadoop.mapreduce.MRJobConfig;
@SuppressWarnings( { "deprecation", "unchecked" })
public class ColumnTruncateTask extends Task<ColumnTruncateWork> implements Serializable,
@@ -142,8 +143,8 @@ public class ColumnTruncateTask extends Task<ColumnTruncateWork> implements Seri
int returnVal = 0;
RunningJob rj = null;
- boolean noName = StringUtils.isEmpty(HiveConf.getVar(job,
- HiveConf.ConfVars.HADOOPJOBNAME));
+
+ boolean noName = StringUtils.isEmpty(job.get(MRJobConfig.JOB_NAME));
String jobName = null;
if (noName && this.getQueryPlan() != null) {
@@ -154,7 +155,7 @@ public class ColumnTruncateTask extends Task<ColumnTruncateWork> implements Seri
if (noName) {
// This is for a special case to ensure unit tests pass
- HiveConf.setVar(job, HiveConf.ConfVars.HADOOPJOBNAME,
+ job.set(MRJobConfig.JOB_NAME,
jobName != null ? jobName : "JOB" + Utilities.randGen.nextInt());
}
http://git-wip-us.apache.org/repos/asf/hive/blob/b9650be8/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java
index 21dcf86..e18117a 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java
@@ -32,7 +32,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.Driver;
-import org.apache.hadoop.hive.ql.io.IOContext;
import org.apache.hadoop.hive.ql.io.IOContextMap;
import org.apache.hadoop.hive.ql.parse.TypeCheckProcFactory;
import org.apache.hadoop.hive.ql.plan.CollectDesc;
@@ -58,6 +57,7 @@ import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.TextInputFormat;
+import org.apache.hadoop.mapreduce.MRJobConfig;
import org.junit.Assert;
import org.junit.Test;
@@ -287,8 +287,7 @@ public class TestOperators extends TestCase {
System.out.println("Testing Map Operator");
// initialize configuration
JobConf hconf = new JobConf(TestOperators.class);
- HiveConf.setVar(hconf, HiveConf.ConfVars.HADOOPMAPFILENAME,
- "hdfs:///testDir/testFile");
+ hconf.set(MRJobConfig.MAP_INPUT_FILE, "hdfs:///testDir/testFile");
IOContextMap.get(hconf).setInputPath(
new Path("hdfs:///testDir/testFile"));
http://git-wip-us.apache.org/repos/asf/hive/blob/b9650be8/ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java b/ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java
index a68049f..2914194 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java
@@ -59,7 +59,6 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
@@ -642,9 +641,7 @@ public class TestRCFile {
RCFileInputFormat inputFormat = new RCFileInputFormat();
JobConf jobconf = new JobConf(cloneConf);
jobconf.set("mapred.input.dir", testDir.toString());
- jobconf.setLong(
- ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZE"),
- fileLen);
+ HiveConf.setLongVar(jobconf, HiveConf.ConfVars.MAPREDMINSPLITSIZE, fileLen);
InputSplit[] splits = inputFormat.getSplits(jobconf, 1);
RCFileRecordReader rr = new RCFileRecordReader(jobconf, (FileSplit)splits[0]);
long lastSync = 0;
@@ -711,9 +708,7 @@ public class TestRCFile {
RCFileInputFormat inputFormat = new RCFileInputFormat();
JobConf jonconf = new JobConf(cloneConf);
jonconf.set("mapred.input.dir", testDir.toString());
- jonconf.setLong(
- ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZE"),
- minSplitSize);
+ HiveConf.setLongVar(jonconf, HiveConf.ConfVars.MAPREDMINSPLITSIZE, minSplitSize);
InputSplit[] splits = inputFormat.getSplits(jonconf, splitNumber);
assertEquals("splits length should be " + splitNumber, splits.length, splitNumber);
int readCount = 0;
http://git-wip-us.apache.org/repos/asf/hive/blob/b9650be8/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
index b8d39d2..9f616ab 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
@@ -1080,8 +1080,8 @@ public class TestInputOutputFormat {
new MockBlock("host0", "host3-2", "host3-3"),
new MockBlock("host4-1", "host4-2", "host4-3"),
new MockBlock("host5-1", "host5-2", "host5-3")));
- conf.setInt(OrcInputFormat.MAX_SPLIT_SIZE, 300);
- conf.setInt(OrcInputFormat.MIN_SPLIT_SIZE, 200);
+ HiveConf.setLongVar(conf, HiveConf.ConfVars.MAPREDMAXSPLITSIZE, 300);
+ HiveConf.setLongVar(conf, HiveConf.ConfVars.MAPREDMINSPLITSIZE, 200);
OrcInputFormat.Context context = new OrcInputFormat.Context(conf);
OrcInputFormat.SplitGenerator splitter =
new OrcInputFormat.SplitGenerator(new OrcInputFormat.SplitInfo(context, fs,
@@ -1104,8 +1104,8 @@ public class TestInputOutputFormat {
assertEquals(1800, result.getStart());
assertEquals(200, result.getLength());
// test min = 0, max = 0 generates each stripe
- conf.setInt(OrcInputFormat.MIN_SPLIT_SIZE, 0);
- conf.setInt(OrcInputFormat.MAX_SPLIT_SIZE, 0);
+ HiveConf.setLongVar(conf, HiveConf.ConfVars.MAPREDMAXSPLITSIZE, 0);
+ HiveConf.setLongVar(conf, HiveConf.ConfVars.MAPREDMINSPLITSIZE, 0);
context = new OrcInputFormat.Context(conf);
splitter = new OrcInputFormat.SplitGenerator(new OrcInputFormat.SplitInfo(context, fs,
AcidUtils.createOriginalObj(null, fs.getFileStatus(new Path("/a/file"))), null, true,
@@ -1129,8 +1129,8 @@ public class TestInputOutputFormat {
new MockBlock("host0", "host3-2", "host3-3"),
new MockBlock("host4-1", "host4-2", "host4-3"),
new MockBlock("host5-1", "host5-2", "host5-3")));
- conf.setInt(OrcInputFormat.MAX_SPLIT_SIZE, 300);
- conf.setInt(OrcInputFormat.MIN_SPLIT_SIZE, 200);
+ HiveConf.setLongVar(conf, HiveConf.ConfVars.MAPREDMAXSPLITSIZE, 300);
+ HiveConf.setLongVar(conf, HiveConf.ConfVars.MAPREDMINSPLITSIZE, 200);
conf.setBoolean(ColumnProjectionUtils.READ_ALL_COLUMNS, false);
conf.set(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR, "0");
OrcInputFormat.Context context = new OrcInputFormat.Context(conf);
@@ -1154,8 +1154,8 @@ public class TestInputOutputFormat {
assertEquals(41867, result.getProjectedColumnsUncompressedSize());
// test min = 0, max = 0 generates each stripe
- conf.setInt(OrcInputFormat.MIN_SPLIT_SIZE, 0);
- conf.setInt(OrcInputFormat.MAX_SPLIT_SIZE, 0);
+ HiveConf.setLongVar(conf, HiveConf.ConfVars.MAPREDMAXSPLITSIZE, 0);
+ HiveConf.setLongVar(conf, HiveConf.ConfVars.MAPREDMINSPLITSIZE, 0);
context = new OrcInputFormat.Context(conf);
splitter = new OrcInputFormat.SplitGenerator(new OrcInputFormat.SplitInfo(context, fs,
fs.getFileStatus(new Path("/a/file")), null, true,
@@ -1174,8 +1174,8 @@ public class TestInputOutputFormat {
}
// single split
- conf.setInt(OrcInputFormat.MIN_SPLIT_SIZE, 100000);
- conf.setInt(OrcInputFormat.MAX_SPLIT_SIZE, 1000);
+ HiveConf.setLongVar(conf, HiveConf.ConfVars.MAPREDMAXSPLITSIZE, 1000);
+ HiveConf.setLongVar(conf, HiveConf.ConfVars.MAPREDMINSPLITSIZE, 100000);
context = new OrcInputFormat.Context(conf);
splitter = new OrcInputFormat.SplitGenerator(new OrcInputFormat.SplitInfo(context, fs,
fs.getFileStatus(new Path("/a/file")), null, true,
http://git-wip-us.apache.org/repos/asf/hive/blob/b9650be8/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcSplitElimination.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcSplitElimination.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcSplitElimination.java
index 02f015c..3560c43 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcSplitElimination.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcSplitElimination.java
@@ -28,6 +28,7 @@ import java.util.List;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
@@ -40,7 +41,6 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrLessThan;
import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
-import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.InputFormat;
@@ -109,8 +109,8 @@ public class TestOrcSplitElimination {
100000, CompressionKind.NONE, 10000, 10000);
writeData(writer);
writer.close();
- conf.set(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZE"), "1000");
- conf.set(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMAXSPLITSIZE"), "5000");
+ HiveConf.setLongVar(conf, HiveConf.ConfVars.MAPREDMINSPLITSIZE, 1000);
+ HiveConf.setLongVar(conf, HiveConf.ConfVars.MAPREDMAXSPLITSIZE, 5000);
InputFormat<?, ?> in = new OrcInputFormat();
FileInputFormat.setInputPaths(conf, testFilePath.toString());
@@ -187,8 +187,8 @@ public class TestOrcSplitElimination {
100000, CompressionKind.NONE, 10000, 10000);
writeData(writer);
writer.close();
- conf.set(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZE"), "1000");
- conf.set(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMAXSPLITSIZE"), "150000");
+ HiveConf.setLongVar(conf, HiveConf.ConfVars.MAPREDMINSPLITSIZE, 1000);
+ HiveConf.setLongVar(conf, HiveConf.ConfVars.MAPREDMAXSPLITSIZE, 150000);
InputFormat<?, ?> in = new OrcInputFormat();
FileInputFormat.setInputPaths(conf, testFilePath.toString());
@@ -276,8 +276,8 @@ public class TestOrcSplitElimination {
100000, CompressionKind.NONE, 10000, 10000);
writeData(writer);
writer.close();
- conf.set(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZE"), "1000");
- conf.set(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMAXSPLITSIZE"), "150000");
+ HiveConf.setLongVar(conf, HiveConf.ConfVars.MAPREDMINSPLITSIZE, 1000);
+ HiveConf.setLongVar(conf, HiveConf.ConfVars.MAPREDMAXSPLITSIZE, 150000);
InputFormat<?, ?> in = new OrcInputFormat();
FileInputFormat.setInputPaths(conf, testFilePath.toString());
http://git-wip-us.apache.org/repos/asf/hive/blob/b9650be8/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
----------------------------------------------------------------------
diff --git a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
index 4da98e4..0b5c092 100644
--- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
+++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
@@ -21,21 +21,16 @@ import java.io.DataInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
-import java.lang.reflect.Constructor;
-import java.lang.reflect.Field;
-import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.net.InetSocketAddress;
import java.net.MalformedURLException;
import java.net.URI;
-import java.net.URL;
import java.nio.ByteBuffer;
import java.security.AccessControlException;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
-import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -1041,25 +1036,6 @@ public class Hadoop23Shims extends HadoopShimsSecure {
}
@Override
- public Map<String, String> getHadoopConfNames() {
- Map<String, String> ret = new HashMap<String, String>();
- ret.put("HADOOPFS", "fs.defaultFS");
- ret.put("HADOOPMAPFILENAME", "mapreduce.map.input.file");
- ret.put("HADOOPMAPREDINPUTDIR", "mapreduce.input.fileinputformat.inputdir");
- ret.put("HADOOPMAPREDINPUTDIRRECURSIVE", "mapreduce.input.fileinputformat.input.dir.recursive");
- ret.put("MAPREDMAXSPLITSIZE", "mapreduce.input.fileinputformat.split.maxsize");
- ret.put("MAPREDMINSPLITSIZE", "mapreduce.input.fileinputformat.split.minsize");
- ret.put("MAPREDMINSPLITSIZEPERNODE", "mapreduce.input.fileinputformat.split.minsize.per.node");
- ret.put("MAPREDMINSPLITSIZEPERRACK", "mapreduce.input.fileinputformat.split.minsize.per.rack");
- ret.put("HADOOPNUMREDUCERS", "mapreduce.job.reduces");
- ret.put("HADOOPJOBNAME", "mapreduce.job.name");
- ret.put("HADOOPSPECULATIVEEXECREDUCERS", "mapreduce.reduce.speculative");
- ret.put("MAPREDSETUPCLEANUPNEEDED", "mapreduce.job.committer.setup.cleanup.needed");
- ret.put("MAPREDTASKCLEANUPNEEDED", "mapreduce.job.committer.task.cleanup.needed");
- return ret;
- }
-
- @Override
public ZeroCopyReaderShim getZeroCopyReader(FSDataInputStream in, ByteBufferPoolShim pool) throws IOException {
if(zeroCopy) {
return ZeroCopyShims.getZeroCopyReader(in, pool);
http://git-wip-us.apache.org/repos/asf/hive/blob/b9650be8/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
----------------------------------------------------------------------
diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
index 47b3caa..f9785ec 100644
--- a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
+++ b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
@@ -27,7 +27,6 @@ import java.security.AccessControlException;
import java.security.NoSuchAlgorithmException;
import java.util.Comparator;
import java.util.List;
-import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
@@ -411,8 +410,6 @@ public interface HadoopShims {
*/
public FileSystem createProxyFileSystem(FileSystem fs, URI uri);
- public Map<String, String> getHadoopConfNames();
-
/**
* Create a shim for DFS storage policy.
*/
http://git-wip-us.apache.org/repos/asf/hive/blob/b9650be8/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java
----------------------------------------------------------------------
diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java
index 0a0f52d..63d48a5 100644
--- a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java
+++ b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java
@@ -294,18 +294,25 @@ public abstract class HadoopShimsSecure implements HadoopShims {
@Override
public CombineFileSplit[] getSplits(JobConf job, int numSplits) throws IOException {
- long minSize = job.getLong(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZE"), 0);
+
+ long minSize =
+ job.getLong(org.apache.hadoop.mapreduce.lib.input.FileInputFormat.SPLIT_MINSIZE, 0);
// For backward compatibility, let the above parameter be used
- if (job.getLong(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZEPERNODE"), 0) == 0) {
+ if (job.getLong(
+ org.apache.hadoop.mapreduce.lib.input.CombineFileInputFormat.SPLIT_MINSIZE_PERNODE,
+ 0) == 0) {
super.setMinSplitSizeNode(minSize);
}
- if (job.getLong(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZEPERRACK"), 0) == 0) {
+ if (job.getLong(
+ org.apache.hadoop.mapreduce.lib.input.CombineFileInputFormat.SPLIT_MINSIZE_PERRACK,
+ 0) == 0) {
super.setMinSplitSizeRack(minSize);
}
- if (job.getLong(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMAXSPLITSIZE"), 0) == 0) {
+ if (job.getLong(org.apache.hadoop.mapreduce.lib.input.FileInputFormat.SPLIT_MAXSIZE,
+ 0) == 0) {
super.setMaxSplitSize(minSize);
}
[04/12] hive git commit: HIVE-12443 Hive Streaming should expose
encoding and serdes for testing (Alan Gates, reviewed by Eugene Koifman)
Posted by om...@apache.org.
HIVE-12443 Hive Streaming should expose encoding and serdes for testing (Alan Gates, reviewed by Eugene Koifman)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6563422f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6563422f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6563422f
Branch: refs/heads/master-fixed
Commit: 6563422f912de382e7abfc7138bffd960c98c086
Parents: d3b5a3f
Author: Alan Gates <ga...@hortonworks.com>
Authored: Thu Nov 19 13:17:49 2015 -0800
Committer: Owen O'Malley <om...@apache.org>
Committed: Fri Nov 20 13:46:22 2015 -0800
----------------------------------------------------------------------
.../hcatalog/streaming/AbstractRecordWriter.java | 19 +++++++++++++++++--
.../hcatalog/streaming/DelimitedInputWriter.java | 5 +++--
.../hcatalog/streaming/StrictJsonWriter.java | 11 +++--------
3 files changed, 23 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/6563422f/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/AbstractRecordWriter.java
----------------------------------------------------------------------
diff --git a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/AbstractRecordWriter.java b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/AbstractRecordWriter.java
index c20e04c..5c15675 100644
--- a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/AbstractRecordWriter.java
+++ b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/AbstractRecordWriter.java
@@ -48,7 +48,7 @@ import java.util.HashSet;
import java.util.List;
-abstract class AbstractRecordWriter implements RecordWriter {
+public abstract class AbstractRecordWriter implements RecordWriter {
static final private Logger LOG = LoggerFactory.getLogger(AbstractRecordWriter.class.getName());
final HiveConf conf;
@@ -110,7 +110,22 @@ abstract class AbstractRecordWriter implements RecordWriter {
return result;
}
- abstract SerDe getSerde() throws SerializationError;
+ /**
+ * Get the SerDe for the Objects created by {@link #encode}. This is public so that test
+ * frameworks can use it.
+ * @return serde
+ * @throws SerializationError
+ */
+ public abstract SerDe getSerde() throws SerializationError;
+
+ /**
+ * Encode a record as an Object that Hive can read with the ObjectInspector associated with the
+ * serde returned by {@link #getSerde}. This is public so that test frameworks can use it.
+ * @param record record to be deserialized
+ * @return deserialized record as an Object
+ * @throws SerializationError
+ */
+ public abstract Object encode(byte[] record) throws SerializationError;
protected abstract ObjectInspector[] getBucketObjectInspectors();
protected abstract StructObjectInspector getRecordObjectInspector();
http://git-wip-us.apache.org/repos/asf/hive/blob/6563422f/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/DelimitedInputWriter.java
----------------------------------------------------------------------
diff --git a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/DelimitedInputWriter.java b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/DelimitedInputWriter.java
index b4d94e3..4f1154e 100644
--- a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/DelimitedInputWriter.java
+++ b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/DelimitedInputWriter.java
@@ -243,7 +243,7 @@ public class DelimitedInputWriter extends AbstractRecordWriter {
}
@Override
- SerDe getSerde() throws SerializationError {
+ public SerDe getSerde() throws SerializationError {
return serde;
}
@@ -260,7 +260,8 @@ public class DelimitedInputWriter extends AbstractRecordWriter {
return bucketObjInspectors;
}
- private Object encode(byte[] record) throws SerializationError {
+ @Override
+ public Object encode(byte[] record) throws SerializationError {
try {
BytesWritable blob = new BytesWritable();
blob.set(record, 0, record.length);
http://git-wip-us.apache.org/repos/asf/hive/blob/6563422f/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/StrictJsonWriter.java
----------------------------------------------------------------------
diff --git a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/StrictJsonWriter.java b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/StrictJsonWriter.java
index 6ab21eb..28ea7d6 100644
--- a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/StrictJsonWriter.java
+++ b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/StrictJsonWriter.java
@@ -86,7 +86,7 @@ public class StrictJsonWriter extends AbstractRecordWriter {
}
@Override
- SerDe getSerde() throws SerializationError {
+ public SerDe getSerde() throws SerializationError {
return serde;
}
@@ -137,13 +137,8 @@ public class StrictJsonWriter extends AbstractRecordWriter {
}
}
- /**
- * Encode Utf8 encoded string bytes using JsonSerde
- * @param utf8StrRecord
- * @return The encoded object
- * @throws SerializationError
- */
- private Object encode(byte[] utf8StrRecord) throws SerializationError {
+ @Override
+ public Object encode(byte[] utf8StrRecord) throws SerializationError {
try {
Text blob = new Text(utf8StrRecord);
return serde.deserialize(blob);
[06/12] hive git commit: HIVE-12418:
HiveHBaseTableInputFormat.getRecordReader() causes Zookeeper connection leak.
(Naveen Gangam via Aihua Xu)
Posted by om...@apache.org.
HIVE-12418: HiveHBaseTableInputFormat.getRecordReader() causes Zookeeper connection leak. (Naveen Gangam via Aihua Xu)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d3b5a3f7
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d3b5a3f7
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d3b5a3f7
Branch: refs/heads/master-fixed
Commit: d3b5a3f763e34e2729edb8cd5c4bc6ddec744ab3
Parents: 3a55074
Author: aihuaxu <ai...@apache.org>
Authored: Thu Nov 19 09:07:50 2015 -0500
Committer: Owen O'Malley <om...@apache.org>
Committed: Fri Nov 20 13:46:22 2015 -0800
----------------------------------------------------------------------
.../hadoop/hive/hbase/HiveHBaseTableInputFormat.java | 10 ++++++++++
1 file changed, 10 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/d3b5a3f7/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java
----------------------------------------------------------------------
diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java
index fff9871..b17714f 100644
--- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java
+++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java
@@ -116,6 +116,7 @@ public class HiveHBaseTableInputFormat extends TableInputFormatBase
@Override
public void close() throws IOException {
recordReader.close();
+ closeTable();
}
@Override
@@ -511,4 +512,13 @@ public class HiveHBaseTableInputFormat extends TableInputFormatBase
closeTable();
}
}
+
+ @Override
+ protected void finalize() throws Throwable {
+ try {
+ closeTable();
+ } finally {
+ super.finalize();
+ }
+ }
}
[05/12] hive git commit: HIVE-11422: Join a ACID table with non-ACID
table fail with MR
Posted by om...@apache.org.
HIVE-11422: Join a ACID table with non-ACID table fail with MR
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3a550743
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3a550743
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3a550743
Branch: refs/heads/master-fixed
Commit: 3a5507437a8447b4c195f888bad670a53444beb5
Parents: 6a3fa6c
Author: Daniel Dai <da...@hortonworks.com>
Authored: Wed Nov 18 17:25:06 2015 -0800
Committer: Owen O'Malley <om...@apache.org>
Committed: Fri Nov 20 13:46:22 2015 -0800
----------------------------------------------------------------------
.../test/resources/testconfiguration.properties | 1 +
.../queries/clientpositive/join_acid_non_acid.q | 24 ++++++++
.../clientpositive/join_acid_non_acid.q.out | 58 ++++++++++++++++++++
3 files changed, 83 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/3a550743/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index a49a301..a33e720 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -28,6 +28,7 @@ minimr.query.files=auto_sortmerge_join_16.q,\
input16_cc.q,\
insert_dir_distcp.q,\
join1.q,\
+ join_acid_non_acid.q,\
leftsemijoin_mr.q,\
list_bucket_dml_10.q,\
load_fs2.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/3a550743/ql/src/test/queries/clientpositive/join_acid_non_acid.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/join_acid_non_acid.q b/ql/src/test/queries/clientpositive/join_acid_non_acid.q
new file mode 100644
index 0000000..43d768f
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/join_acid_non_acid.q
@@ -0,0 +1,24 @@
+set hive.support.concurrency=true;
+set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
+
+CREATE TABLE orc_update_table (k1 INT, f1 STRING, op_code STRING)
+CLUSTERED BY (k1) INTO 2 BUCKETS
+STORED AS ORC TBLPROPERTIES("transactional"="true");
+
+INSERT INTO TABLE orc_update_table VALUES (1, 'a', 'I');
+
+CREATE TABLE orc_table (k1 INT, f1 STRING)
+CLUSTERED BY (k1) SORTED BY (k1) INTO 2 BUCKETS
+STORED AS ORC;
+
+INSERT OVERWRITE TABLE orc_table VALUES (1, 'x');
+
+set hive.cbo.enable=true;
+SET hive.execution.engine=mr;
+SET hive.auto.convert.join=false;
+SET hive.input.format=org.apache.hadoop.hive.ql.io.CombineHiveInputFormat;
+SET hive.conf.validation=false;
+SET hive.doing.acid=false;
+
+SELECT t1.*, t2.* FROM orc_table t1
+JOIN orc_update_table t2 ON t1.k1=t2.k1 ORDER BY t1.k1;
http://git-wip-us.apache.org/repos/asf/hive/blob/3a550743/ql/src/test/results/clientpositive/join_acid_non_acid.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/join_acid_non_acid.q.out b/ql/src/test/results/clientpositive/join_acid_non_acid.q.out
new file mode 100644
index 0000000..4905351
--- /dev/null
+++ b/ql/src/test/results/clientpositive/join_acid_non_acid.q.out
@@ -0,0 +1,58 @@
+PREHOOK: query: CREATE TABLE orc_update_table (k1 INT, f1 STRING, op_code STRING)
+CLUSTERED BY (k1) INTO 2 BUCKETS
+STORED AS ORC TBLPROPERTIES("transactional"="true")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@orc_update_table
+POSTHOOK: query: CREATE TABLE orc_update_table (k1 INT, f1 STRING, op_code STRING)
+CLUSTERED BY (k1) INTO 2 BUCKETS
+STORED AS ORC TBLPROPERTIES("transactional"="true")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@orc_update_table
+PREHOOK: query: INSERT INTO TABLE orc_update_table VALUES (1, 'a', 'I')
+PREHOOK: type: QUERY
+PREHOOK: Input: default@values__tmp__table__1
+PREHOOK: Output: default@orc_update_table
+POSTHOOK: query: INSERT INTO TABLE orc_update_table VALUES (1, 'a', 'I')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@values__tmp__table__1
+POSTHOOK: Output: default@orc_update_table
+POSTHOOK: Lineage: orc_update_table.f1 SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+POSTHOOK: Lineage: orc_update_table.k1 EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+POSTHOOK: Lineage: orc_update_table.op_code SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ]
+PREHOOK: query: CREATE TABLE orc_table (k1 INT, f1 STRING)
+CLUSTERED BY (k1) SORTED BY (k1) INTO 2 BUCKETS
+STORED AS ORC
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@orc_table
+POSTHOOK: query: CREATE TABLE orc_table (k1 INT, f1 STRING)
+CLUSTERED BY (k1) SORTED BY (k1) INTO 2 BUCKETS
+STORED AS ORC
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@orc_table
+PREHOOK: query: INSERT OVERWRITE TABLE orc_table VALUES (1, 'x')
+PREHOOK: type: QUERY
+PREHOOK: Input: default@values__tmp__table__2
+PREHOOK: Output: default@orc_table
+POSTHOOK: query: INSERT OVERWRITE TABLE orc_table VALUES (1, 'x')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@values__tmp__table__2
+POSTHOOK: Output: default@orc_table
+POSTHOOK: Lineage: orc_table.f1 SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+POSTHOOK: Lineage: orc_table.k1 EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: SELECT t1.*, t2.* FROM orc_table t1
+JOIN orc_update_table t2 ON t1.k1=t2.k1 ORDER BY t1.k1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_table
+PREHOOK: Input: default@orc_update_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT t1.*, t2.* FROM orc_table t1
+JOIN orc_update_table t2 ON t1.k1=t2.k1 ORDER BY t1.k1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_table
+POSTHOOK: Input: default@orc_update_table
+#### A masked pattern was here ####
+1 x 1 a I