You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by mg...@apache.org on 2019/11/20 10:53:55 UTC
[hive] branch master updated: HIVE-22329 Create DriverContext
(Miklos Gergely, reviewed by Zoltan Haindrich and Krisztian Kasa)
This is an automated email from the ASF dual-hosted git repository.
mgergely pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new edaf4d8 HIVE-22329 Create DriverContext (Miklos Gergely, reviewed by Zoltan Haindrich and Krisztian Kasa)
edaf4d8 is described below
commit edaf4d86d0677f5d8a3f9ff14a6eaef905d04e50
Author: miklosgergely <mg...@cloudera.com>
AuthorDate: Sat Oct 19 11:07:10 2019 +0200
HIVE-22329 Create DriverContext (Miklos Gergely, reviewed by Zoltan Haindrich and Krisztian Kasa)
---
.../mapreduce/TestHCatMultiOutputFormat.java | 5 +-
.../hive/ql/exec/spark/TestSparkStatistics.java | 11 +-
.../hive/ql/parse/TestReplicationScenarios.java | 4 +-
ql/src/java/org/apache/hadoop/hive/ql/Driver.java | 731 ++++++++++-----------
.../org/apache/hadoop/hive/ql/DriverContext.java | 351 +++++-----
.../org/apache/hadoop/hive/ql/DriverUtils.java | 4 +-
.../org/apache/hadoop/hive/ql/QueryProperties.java | 1 -
.../hive/ql/{DriverContext.java => TaskQueue.java} | 38 +-
.../hadoop/hive/ql/ddl/DDLOperationContext.java | 14 +-
.../org/apache/hadoop/hive/ql/ddl/DDLTask.java | 20 +-
.../ddl/database/lock/LockDatabaseOperation.java | 2 +-
.../database/unlock/UnlockDatabaseOperation.java | 2 +-
.../hive/ql/ddl/table/lock/LockTableOperation.java | 2 +-
.../hive/ql/ddl/table/lock/ShowLocksOperation.java | 2 +-
.../ql/ddl/table/lock/UnlockTableOperation.java | 2 +-
.../ql/ddl/table/misc/TruncateTableOperation.java | 8 +-
.../table/storage/AlterTableArchiveOperation.java | 2 +-
.../storage/AlterTableConcatenateOperation.java | 17 +-
.../storage/AlterTableUnarchiveOperation.java | 2 +-
.../update/MaterializedViewUpdateOperation.java | 2 +-
.../hadoop/hive/ql/exec/ColumnStatsUpdateTask.java | 12 +-
.../hadoop/hive/ql/exec/ConditionalTask.java | 13 +-
.../org/apache/hadoop/hive/ql/exec/CopyTask.java | 13 +-
.../hive/ql/exec/DependencyCollectionTask.java | 3 +-
.../hadoop/hive/ql/exec/ExplainSQRewriteTask.java | 4 +-
.../apache/hadoop/hive/ql/exec/ExplainTask.java | 3 +-
.../org/apache/hadoop/hive/ql/exec/ExportTask.java | 3 +-
.../org/apache/hadoop/hive/ql/exec/FetchTask.java | 13 +-
.../org/apache/hadoop/hive/ql/exec/MoveTask.java | 19 +-
.../apache/hadoop/hive/ql/exec/ReplCopyTask.java | 5 +-
.../apache/hadoop/hive/ql/exec/ReplTxnTask.java | 69 +-
.../org/apache/hadoop/hive/ql/exec/StatsTask.java | 15 +-
.../java/org/apache/hadoop/hive/ql/exec/Task.java | 37 +-
.../org/apache/hadoop/hive/ql/exec/TaskRunner.java | 12 +-
.../apache/hadoop/hive/ql/exec/mr/ExecDriver.java | 24 +-
.../apache/hadoop/hive/ql/exec/mr/MapRedTask.java | 13 +-
.../hadoop/hive/ql/exec/mr/MapredLocalTask.java | 25 +-
.../ql/exec/repl/ExternalTableCopyTaskBuilder.java | 3 +-
.../hadoop/hive/ql/exec/repl/ReplDumpTask.java | 3 +-
.../hadoop/hive/ql/exec/repl/ReplLoadTask.java | 39 +-
.../hadoop/hive/ql/exec/repl/ReplStateLogTask.java | 3 +-
.../incremental/IncrementalLoadTasksBuilder.java | 11 +-
.../exec/schq/ScheduledQueryMaintenanceTask.java | 3 +-
.../hadoop/hive/ql/exec/spark/HiveSparkClient.java | 9 +-
.../hive/ql/exec/spark/LocalHiveSparkClient.java | 14 +-
.../hive/ql/exec/spark/RemoteHiveSparkClient.java | 18 +-
.../hadoop/hive/ql/exec/spark/SparkTask.java | 16 +-
.../hive/ql/exec/spark/session/SparkSession.java | 7 +-
.../ql/exec/spark/session/SparkSessionImpl.java | 7 +-
.../apache/hadoop/hive/ql/exec/tez/TezTask.java | 5 +-
.../ql/hooks/PostExecWMEventsSummaryPrinter.java | 2 +-
.../hadoop/hive/ql/io/merge/MergeFileTask.java | 12 +-
.../ql/io/rcfile/truncate/ColumnTruncateTask.java | 14 +-
.../hive/ql/parse/AcidExportSemanticAnalyzer.java | 6 +-
.../hive/ql/parse/ExplainSemanticAnalyzer.java | 2 +-
.../apache/hadoop/hive/ql/plan/BasicStatsWork.java | 2 +-
.../apache/hadoop/hive/ql/exec/TestExecDriver.java | 8 +-
.../hadoop/hive/ql/exec/mr/TestMapRedTask.java | 9 +-
.../hadoop/hive/ql/exec/spark/TestSparkTask.java | 12 +-
.../hive/ql/parse/TestMacroSemanticAnalyzer.java | 4 +-
.../ql/parse/TestUpdateDeleteSemanticAnalyzer.java | 2 +-
61 files changed, 787 insertions(+), 927 deletions(-)
diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java
index 8a8a326..c62d356 100644
--- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java
+++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java
@@ -41,7 +41,6 @@ import org.apache.hadoop.hive.metastore.api.SerDeInfo;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
-import org.apache.hadoop.hive.ql.CompilationOpContext;
import org.apache.hadoop.hive.ql.QueryState;
import org.apache.hadoop.hive.ql.exec.FetchTask;
import org.apache.hadoop.hive.ql.exec.Utilities;
@@ -386,7 +385,9 @@ public class TestHCatMultiOutputFormat {
}
FetchTask task = new FetchTask();
task.setWork(work);
- task.initialize(queryState, null, null, new CompilationOpContext());
+ conf.set("_hive.hdfs.session.path", "path");
+ conf.set("_hive.local.session.path", "path");
+ task.initialize(queryState, null, null, new org.apache.hadoop.hive.ql.Context(conf));
task.fetch(temp);
for (String str : temp) {
results.add(str.replace("\t", ","));
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkStatistics.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkStatistics.java
index 205b0fd..2b72d09 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkStatistics.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkStatistics.java
@@ -22,7 +22,7 @@ import com.google.common.collect.Lists;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.Driver;
-import org.apache.hadoop.hive.ql.DriverContext;
+import org.apache.hadoop.hive.ql.TaskQueue;
import org.apache.hadoop.hive.ql.QueryState;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.exec.spark.Statistic.SparkStatistic;
@@ -68,12 +68,11 @@ public class TestSparkStatistics {
SparkTask sparkTask = sparkTasks.get(0);
- DriverContext driverCxt = new DriverContext(driver.getContext());
- driverCxt.prepare(driver.getPlan());
+ TaskQueue taskQueue = new TaskQueue(driver.getContext());
+ taskQueue.prepare(driver.getPlan());
- sparkTask.initialize(driver.getQueryState(), driver.getPlan(), driverCxt, driver.getContext()
- .getOpContext());
- Assert.assertEquals(0, sparkTask.execute(driverCxt));
+ sparkTask.initialize(driver.getQueryState(), driver.getPlan(), taskQueue, driver.getContext());
+ Assert.assertEquals(0, sparkTask.execute());
Assert.assertNotNull(sparkTask.getSparkStatistics());
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
index e0010b8..823fabb 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
@@ -53,7 +53,7 @@ import org.apache.hadoop.hive.metastore.messaging.event.filters.AndFilter;
import org.apache.hadoop.hive.metastore.messaging.event.filters.DatabaseAndTableFilter;
import org.apache.hadoop.hive.metastore.messaging.event.filters.EventBoundaryFilter;
import org.apache.hadoop.hive.metastore.messaging.event.filters.MessageFormatFilter;
-import org.apache.hadoop.hive.ql.DriverContext;
+import org.apache.hadoop.hive.ql.TaskQueue;
import org.apache.hadoop.hive.metastore.messaging.json.JSONMessageEncoder;
import org.apache.hadoop.hive.metastore.messaging.json.gzip.GzipJSONMessageEncoder;
import org.apache.hadoop.hive.ql.DriverFactory;
@@ -399,7 +399,7 @@ public class TestReplicationScenarios {
null, null, isIncrementalDump, Long.valueOf(tuple.lastReplId),
Collections.emptyList());
Task replLoadTask = TaskFactory.get(replLoadWork, confTemp);
- replLoadTask.initialize(null, null, new DriverContext(driver.getContext()), null);
+ replLoadTask.initialize(null, null, new TaskQueue(driver.getContext()), driver.getContext());
replLoadTask.executeTask(null);
Hive.closeCurrent();
return replLoadWork.getRootTask();
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index a920fcf..bb41c15 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -133,78 +133,33 @@ import com.google.common.collect.ImmutableMap;
public class Driver implements IDriver {
- static final private String CLASS_NAME = Driver.class.getName();
+ private static final String CLASS_NAME = Driver.class.getName();
private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME);
- static final private LogHelper console = new LogHelper(LOG);
+ private static final LogHelper CONSOLE = new LogHelper(LOG);
private static final int SHUTDOWN_HOOK_PRIORITY = 0;
- private final QueryInfo queryInfo;
private Runnable shutdownRunner = null;
private int maxRows = 100;
private ByteStream.Output bos = new ByteStream.Output();
- private final HiveConf conf;
private DataInput resStream;
private Context ctx;
- private DriverContext driverCxt;
- private QueryPlan plan;
- private Schema schema;
-
- private FetchTask fetchTask;
- private List<HiveLock> hiveLocks = new ArrayList<HiveLock>();
-
- // A limit on the number of threads that can be launched
- private int maxthreads;
-
- private String userName;
+ private final DriverContext driverContext;
+ private TaskQueue taskQueue;
+ private final List<HiveLock> hiveLocks = new ArrayList<HiveLock>();
// HS2 operation handle guid string
private String operationId;
- // For WebUI. Kept alive after queryPlan is freed.
- private final QueryDisplay queryDisplay = new QueryDisplay();
private DriverState driverState = new DriverState();
- // Query specific info
- private final QueryState queryState;
-
- // Query hooks that execute before compilation and after execution
- private HookRunner hookRunner;
-
- // Transaction manager the Driver has been initialized with (can be null).
- // If this is set then this Transaction manager will be used during query
- // compilation/execution rather than using the current session's transaction manager.
- // This might be needed in a situation where a Driver is nested within an already
- // running Driver/query - the nested Driver requires a separate transaction manager
- // so as not to conflict with the outer Driver/query which is using the session
- // transaction manager.
- private final HiveTxnManager initTxnMgr;
-
- // Transaction manager used for the query. This will be set at compile time based on
- // either initTxnMgr or from the SessionState, in that order.
- private HiveTxnManager queryTxnMgr;
- private TxnType queryTxnType = TxnType.DEFAULT;
- private StatsSource statsSource;
-
- // Boolean to store information about whether valid txn list was generated
- // for current query.
- private boolean validTxnListsGenerated;
-
- private CacheUsage cacheUsage;
- private CacheEntry usedCacheEntry;
- private ValidWriteIdList compactionWriteIds = null;
- private long compactorTxnId = 0;
-
- private Context backupContext = null;
- private boolean retrial = false;
-
private boolean checkConcurrency() {
- return conf.getBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY);
+ return driverContext.getConf().getBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY);
}
@Override
public Schema getSchema() {
- return schema;
+ return driverContext.getSchema();
}
public Schema getExplainSchema() {
@@ -304,13 +259,9 @@ public class Driver implements IDriver {
this(queryState, userName, queryInfo, null);
}
- public Driver(QueryState queryState, String userName, QueryInfo queryInfo, HiveTxnManager txnMgr) {
- this.queryState = queryState;
- this.conf = queryState.getConf();
- this.userName = userName;
- this.hookRunner = new HookRunner(conf, console);
- this.queryInfo = queryInfo;
- this.initTxnMgr = txnMgr;
+ public Driver(QueryState queryState, String userName, QueryInfo queryInfo, HiveTxnManager txnManager) {
+ driverContext = new DriverContext(queryState, queryInfo, userName, new HookRunner(queryState.getConf(), CONSOLE),
+ txnManager);
}
/**
@@ -359,13 +310,13 @@ public class Driver implements IDriver {
public Map<String, String> getHiveVariable() {
return SessionState.get().getHiveVariables();
}
- }).substitute(conf, command);
+ }).substitute(driverContext.getConf(), command);
String queryStr = command;
try {
// command should be redacted to avoid to logging sensitive data
- queryStr = HookUtils.redactLogString(conf, command);
+ queryStr = HookUtils.redactLogString(driverContext.getConf(), command);
} catch (Exception e) {
LOG.warn("WARNING! Query command could not be redacted." + e);
}
@@ -383,8 +334,8 @@ public class Driver implements IDriver {
DriverState.setDriverState(driverState);
- final String queryId = Strings.isNullOrEmpty(queryState.getQueryId()) ?
- QueryPlan.makeQueryId() : queryState.getQueryId();
+ final String queryId = Strings.isNullOrEmpty(driverContext.getQueryState().getQueryId()) ?
+ QueryPlan.makeQueryId() : driverContext.getQueryState().getQueryId();
SparkSession ss = SessionState.get().getSparkSession();
if (ss != null) {
@@ -395,12 +346,12 @@ public class Driver implements IDriver {
setTriggerContext(queryId);
}
//save some info for webUI for use after plan is freed
- this.queryDisplay.setQueryStr(queryStr);
- this.queryDisplay.setQueryId(queryId);
+ driverContext.getQueryDisplay().setQueryStr(queryStr);
+ driverContext.getQueryDisplay().setQueryId(queryId);
LOG.info("Compiling command(queryId=" + queryId + "): " + queryStr);
- conf.setQueryString(queryStr);
+ driverContext.getConf().setQueryString(queryStr);
// FIXME: sideeffect will leave the last query set at the session level
if (SessionState.get() != null) {
SessionState.get().getConf().setQueryString(queryStr);
@@ -415,19 +366,19 @@ public class Driver implements IDriver {
checkInterrupted("before parsing and analysing the query", null, null);
if (ctx == null) {
- ctx = new Context(conf);
+ ctx = new Context(driverContext.getConf());
setTriggerContext(queryId);
}
- ctx.setHiveTxnManager(queryTxnMgr);
- ctx.setStatsSource(statsSource);
+ ctx.setHiveTxnManager(driverContext.getTxnManager());
+ ctx.setStatsSource(driverContext.getStatsSource());
ctx.setCmd(command);
ctx.setHDFSCleanup(true);
perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.PARSE);
// Trigger query hook before compilation
- hookRunner.runBeforeParseHook(command);
+ driverContext.getHookRunner().runBeforeParseHook(command);
ASTNode tree;
try {
@@ -436,11 +387,11 @@ public class Driver implements IDriver {
parseError = true;
throw e;
} finally {
- hookRunner.runAfterParseHook(command, parseError);
+ driverContext.getHookRunner().runAfterParseHook(command, parseError);
}
perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.PARSE);
- hookRunner.runBeforeCompileHook(command);
+ driverContext.getHookRunner().runBeforeCompileHook(command);
// clear CurrentFunctionsInUse set, to capture new set of functions
// that SemanticAnalyzer finds are in use
SessionState.get().getCurrentFunctionsInUse().clear();
@@ -452,30 +403,30 @@ public class Driver implements IDriver {
// because at that point we need access to the objects.
Hive.get().getMSC().flushCache();
- backupContext = new Context(ctx);
- boolean executeHooks = hookRunner.hasPreAnalyzeHooks();
+ driverContext.setBackupContext(new Context(ctx));
+ boolean executeHooks = driverContext.getHookRunner().hasPreAnalyzeHooks();
HiveSemanticAnalyzerHookContext hookCtx = new HiveSemanticAnalyzerHookContextImpl();
if (executeHooks) {
- hookCtx.setConf(conf);
- hookCtx.setUserName(userName);
+ hookCtx.setConf(driverContext.getConf());
+ hookCtx.setUserName(driverContext.getUserName());
hookCtx.setIpAddress(SessionState.get().getUserIpAddress());
hookCtx.setCommand(command);
- hookCtx.setHiveOperation(queryState.getHiveOperation());
+ hookCtx.setHiveOperation(driverContext.getQueryState().getHiveOperation());
- tree = hookRunner.runPreAnalyzeHooks(hookCtx, tree);
+ tree = driverContext.getHookRunner().runPreAnalyzeHooks(hookCtx, tree);
}
// Do semantic analysis and plan generation
- BaseSemanticAnalyzer sem = SemanticAnalyzerFactory.get(queryState, tree);
+ BaseSemanticAnalyzer sem = SemanticAnalyzerFactory.get(driverContext.getQueryState(), tree);
- if (!retrial) {
- if ((queryState.getHiveOperation() != null)
- && queryState.getHiveOperation().equals(HiveOperation.REPLDUMP)) {
- setLastReplIdForDump(queryState.getConf());
+ if (!driverContext.isRetrial()) {
+ if ((driverContext.getQueryState().getHiveOperation() != null) &&
+ driverContext.getQueryState().getHiveOperation().equals(HiveOperation.REPLDUMP)) {
+ setLastReplIdForDump(driverContext.getQueryState().getConf());
}
- queryTxnType = AcidUtils.getTxnType(tree);
- openTransaction(queryTxnType);
+ driverContext.setTxnType(AcidUtils.getTxnType(tree));
+ openTransaction(driverContext.getTxnType());
generateValidTxnList();
}
@@ -484,14 +435,14 @@ public class Driver implements IDriver {
if (executeHooks) {
hookCtx.update(sem);
- hookRunner.runPostAnalyzeHooks(hookCtx, sem.getAllRootTasks());
+ driverContext.getHookRunner().runPostAnalyzeHooks(hookCtx, sem.getAllRootTasks());
}
- LOG.info("Semantic Analysis Completed (retrial = {})", retrial);
+ LOG.info("Semantic Analysis Completed (retrial = {})", driverContext.isRetrial());
// Retrieve information about cache usage for the query.
- if (conf.getBoolVar(HiveConf.ConfVars.HIVE_QUERY_RESULTS_CACHE_ENABLED)) {
- cacheUsage = sem.getCacheUsage();
+ if (driverContext.getConf().getBoolVar(HiveConf.ConfVars.HIVE_QUERY_RESULTS_CACHE_ENABLED)) {
+ driverContext.setCacheUsage(sem.getCacheUsage());
}
// validate the plan
@@ -501,51 +452,52 @@ public class Driver implements IDriver {
checkInterrupted("after analyzing query.", null, null);
// get the output schema
- schema = getSchema(sem, conf);
- plan = new QueryPlan(queryStr, sem, queryDisplay.getQueryStartTime(), queryId,
- queryState.getHiveOperation(), schema);
+ driverContext.setSchema(getSchema(sem, driverContext.getConf()));
+ QueryPlan plan = new QueryPlan(queryStr, sem, driverContext.getQueryDisplay().getQueryStartTime(), queryId,
+ driverContext.getQueryState().getHiveOperation(), driverContext.getSchema());
// save the optimized plan and sql for the explain
plan.setOptimizedCBOPlan(ctx.getCalcitePlan());
plan.setOptimizedQueryString(ctx.getOptimizedSql());
+ driverContext.setPlan(plan);
- conf.set("mapreduce.workflow.id", "hive_" + queryId);
- conf.set("mapreduce.workflow.name", queryStr);
+ driverContext.getConf().set("mapreduce.workflow.id", "hive_" + queryId);
+ driverContext.getConf().set("mapreduce.workflow.name", queryStr);
// initialize FetchTask right here
if (plan.getFetchTask() != null) {
- plan.getFetchTask().initialize(queryState, plan, null, ctx.getOpContext());
+ plan.getFetchTask().initialize(driverContext.getQueryState(), plan, null, ctx);
}
//do the authorization check
if (!sem.skipAuthorization() &&
- HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_AUTHORIZATION_ENABLED)) {
+ HiveConf.getBoolVar(driverContext.getConf(), HiveConf.ConfVars.HIVE_AUTHORIZATION_ENABLED)) {
try {
perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.DO_AUTHORIZATION);
// Authorization check for kill query will be in KillQueryImpl
// As both admin or operation owner can perform the operation.
// Which is not directly supported in authorizer
- if (queryState.getHiveOperation() != HiveOperation.KILL_QUERY) {
- CommandAuthorizer.doAuthorization(queryState.getHiveOperation(), sem, command);
+ if (driverContext.getQueryState().getHiveOperation() != HiveOperation.KILL_QUERY) {
+ CommandAuthorizer.doAuthorization(driverContext.getQueryState().getHiveOperation(), sem, command);
}
} catch (AuthorizationException authExp) {
- console.printError("Authorization failed:" + authExp.getMessage() + ". Use SHOW GRANT to get more details.");
+ CONSOLE.printError("Authorization failed:" + authExp.getMessage() + ". Use SHOW GRANT to get more details.");
throw createProcessorException(403, authExp.getMessage(), "42000", null);
} finally {
perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.DO_AUTHORIZATION);
}
}
- if (conf.getBoolVar(ConfVars.HIVE_LOG_EXPLAIN_OUTPUT)
- || conf.getBoolVar(ConfVars.HIVE_SERVER2_WEBUI_EXPLAIN_OUTPUT)) {
+ if (driverContext.getConf().getBoolVar(ConfVars.HIVE_LOG_EXPLAIN_OUTPUT)
+ || driverContext.getConf().getBoolVar(ConfVars.HIVE_SERVER2_WEBUI_EXPLAIN_OUTPUT)) {
String explainOutput = getExplainOutput(sem, plan, tree);
if (explainOutput != null) {
- if (conf.getBoolVar(ConfVars.HIVE_LOG_EXPLAIN_OUTPUT)) {
+ if (driverContext.getConf().getBoolVar(ConfVars.HIVE_LOG_EXPLAIN_OUTPUT)) {
LOG.info("EXPLAIN output for queryid " + queryId + " : " + explainOutput);
}
- if (conf.isWebUiQueryInfoCacheEnabled()
- && conf.getBoolVar(ConfVars.HIVE_SERVER2_WEBUI_EXPLAIN_OUTPUT)) {
- queryDisplay.setExplainPlan(explainOutput);
+ if (driverContext.getConf().isWebUiQueryInfoCacheEnabled()
+ && driverContext.getConf().getBoolVar(ConfVars.HIVE_SERVER2_WEBUI_EXPLAIN_OUTPUT)) {
+ driverContext.getQueryDisplay().setExplainPlan(explainOutput);
}
}
}
@@ -572,14 +524,14 @@ public class Driver implements IDriver {
errorMessage += ". Failed command: " + queryStr;
}
- console.printError(errorMessage, "\n" + StringUtils.stringifyException(e));
+ CONSOLE.printError(errorMessage, "\n" + StringUtils.stringifyException(e));
throw createProcessorException(error.getErrorCode(), errorMessage, error.getSQLState(), e);
} finally {
// Trigger post compilation hook. Note that if the compilation fails here then
// before/after execution hook will never be executed.
if (!parseError) {
try {
- hookRunner.runAfterCompilationHook(command, compileError);
+ driverContext.getHookRunner().runAfterCompilationHook(command, compileError);
} catch (Exception e) {
LOG.warn("Failed when invoking query after-compilation hook.", e);
}
@@ -587,7 +539,7 @@ public class Driver implements IDriver {
double duration = perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.COMPILE)/1000.00;
ImmutableMap<String, Long> compileHMSTimings = dumpMetaCallTimingWithoutEx("compilation");
- queryDisplay.setHmsTimings(QueryDisplay.Phase.COMPILATION, compileHMSTimings);
+ driverContext.getQueryDisplay().setHmsTimings(QueryDisplay.Phase.COMPILATION, compileHMSTimings);
boolean isInterrupted = driverState.isAborted();
if (isInterrupted && !deferClose) {
@@ -607,15 +559,14 @@ public class Driver implements IDriver {
private void createTransactionManager() throws CommandProcessorException {
try {
// Initialize the transaction manager. This must be done before analyze is called.
- if (initTxnMgr != null) {
- queryTxnMgr = initTxnMgr;
- } else {
- queryTxnMgr = SessionState.get().initTxnMgr(conf);
- }
- if (queryTxnMgr instanceof Configurable) {
- ((Configurable) queryTxnMgr).setConf(conf);
+ HiveTxnManager queryTxnManager = (driverContext.getInitTxnManager() != null) ?
+ driverContext.getInitTxnManager() : SessionState.get().initTxnMgr(driverContext.getConf());
+
+ if (queryTxnManager instanceof Configurable) {
+ ((Configurable) queryTxnManager).setConf(driverContext.getConf());
}
- queryState.setTxnManager(queryTxnMgr);
+ driverContext.setTxnManager(queryTxnManager);
+ driverContext.getQueryState().setTxnManager(queryTxnManager);
// In case when user Ctrl-C twice to kill Hive CLI JVM, we want to release locks
// if compile is being called multiple times, clear the old shutdownhook
@@ -624,7 +575,7 @@ public class Driver implements IDriver {
@Override
public void run() {
try {
- releaseLocksAndCommitOrRollback(false, queryTxnMgr);
+ releaseLocksAndCommitOrRollback(false, driverContext.getTxnManager());
} catch (LockException e) {
LOG.warn("Exception when releasing locks in ShutdownHook for Driver: " +
e.getMessage());
@@ -636,7 +587,7 @@ public class Driver implements IDriver {
ErrorMsg error = ErrorMsg.getErrorMsg(e.getMessage());
String errorMessage = "FAILED: " + e.getClass().getSimpleName() + " [Error " + error.getErrorCode() + "]:";
- console.printError(errorMessage, "\n" + StringUtils.stringifyException(e));
+ CONSOLE.printError(errorMessage, "\n" + StringUtils.stringifyException(e));
throw createProcessorException(error.getErrorCode(), errorMessage, error.getSQLState(), e);
}
}
@@ -647,12 +598,12 @@ public class Driver implements IDriver {
// required.
private boolean isValidTxnListState() throws LockException {
// 1) Get valid txn list.
- String txnString = conf.get(ValidTxnList.VALID_TXNS_KEY);
+ String txnString = driverContext.getConf().get(ValidTxnList.VALID_TXNS_KEY);
if (txnString == null) {
// Not a transactional op, nothing more to do
return true;
}
- ValidTxnList currentTxnList = queryTxnMgr.getValidTxns();
+ ValidTxnList currentTxnList = driverContext.getTxnManager().getValidTxns();
String currentTxnString = currentTxnList.toString();
if (currentTxnString.equals(txnString)) {
// Still valid, nothing more to do
@@ -694,16 +645,16 @@ public class Driver implements IDriver {
}
}
// 3) Get txn tables that are being written
- String txnWriteIdListStr = conf.get(ValidTxnWriteIdList.VALID_TABLES_WRITEIDS_KEY);
+ String txnWriteIdListStr = driverContext.getConf().get(ValidTxnWriteIdList.VALID_TABLES_WRITEIDS_KEY);
if (txnWriteIdListStr == null || txnWriteIdListStr.length() == 0) {
// Nothing to check
return true;
}
ValidTxnWriteIdList txnWriteIdList = new ValidTxnWriteIdList(txnWriteIdListStr);
- Map<String, Table> writtenTables = getWrittenTables(plan);
+ Map<String, Table> writtenTables = getWrittenTables(driverContext.getPlan());
ValidTxnWriteIdList currentTxnWriteIds =
- queryTxnMgr.getValidWriteIds(
+ driverContext.getTxnManager().getValidWriteIds(
writtenTables.entrySet().stream()
.filter(e -> AcidUtils.isTransactionalTable(e.getValue()))
.map(Map.Entry::getKey)
@@ -744,10 +695,10 @@ public class Driver implements IDriver {
// query info is created by SQLOperation which will have start time of the operation. When JDBC Statement is not
// used queryInfo will be null, in which case we take creation of Driver instance as query start time (which is also
// the time when query display object is created)
- if (queryInfo != null) {
- queryStartTime = queryInfo.getBeginTime();
+ if (driverContext.getQueryInfo() != null) {
+ queryStartTime = driverContext.getQueryInfo().getBeginTime();
} else {
- queryStartTime = queryDisplay.getQueryStartTime();
+ queryStartTime = driverContext.getQueryDisplay().getQueryStartTime();
}
WmContext wmContext = new WmContext(queryStartTime, queryId);
ctx.setWmContext(wmContext);
@@ -770,9 +721,10 @@ public class Driver implements IDriver {
}
private void openTransaction(TxnType txnType) throws LockException, CommandProcessorException {
- if (checkConcurrency() && startImplicitTxn(queryTxnMgr) && !queryTxnMgr.isTxnOpen()) {
+ if (checkConcurrency() && startImplicitTxn(driverContext.getTxnManager()) &&
+ !driverContext.getTxnManager().isTxnOpen()) {
String userFromUGI = getUserFromUGI();
- queryTxnMgr.openTxn(ctx, userFromUGI, txnType);
+ driverContext.getTxnManager().openTxn(ctx, userFromUGI, txnType);
}
}
@@ -781,12 +733,12 @@ public class Driver implements IDriver {
// compilation and processing. We only do this if 1) a transaction
// was already opened and 2) the list has not been recorded yet,
// e.g., by an explicit open transaction command.
- validTxnListsGenerated = false;
- String currentTxnString = conf.get(ValidTxnList.VALID_TXNS_KEY);
- if (queryTxnMgr.isTxnOpen() && (currentTxnString == null || currentTxnString.isEmpty())) {
+ driverContext.setValidTxnListsGenerated(false);
+ String currentTxnString = driverContext.getConf().get(ValidTxnList.VALID_TXNS_KEY);
+ if (driverContext.getTxnManager().isTxnOpen() && (currentTxnString == null || currentTxnString.isEmpty())) {
try {
- recordValidTxns(queryTxnMgr);
- validTxnListsGenerated = true;
+ recordValidTxns(driverContext.getTxnManager());
+ driverContext.setValidTxnListsGenerated(true);
} catch (LockException e) {
LOG.error("Exception while acquiring valid txn list", e);
throw e;
@@ -797,33 +749,34 @@ public class Driver implements IDriver {
private boolean startImplicitTxn(HiveTxnManager txnManager) throws LockException {
boolean shouldOpenImplicitTxn = !ctx.isExplainPlan();
//this is dumb. HiveOperation is not always set. see HIVE-16447/HIVE-16443
- switch (queryState.getHiveOperation() == null ? HiveOperation.QUERY : queryState.getHiveOperation()) {
- case COMMIT:
- case ROLLBACK:
- if(!txnManager.isTxnOpen()) {
- throw new LockException(null, ErrorMsg.OP_NOT_ALLOWED_WITHOUT_TXN, queryState.getHiveOperation().getOperationName());
- }
- case SWITCHDATABASE:
- case SET_AUTOCOMMIT:
- /**
- * autocommit is here for completeness. TM doesn't use it. If we want to support JDBC
- * semantics (or any other definition of autocommit) it should be done at session level.
- */
- case SHOWDATABASES:
- case SHOWTABLES:
- case SHOWCOLUMNS:
- case SHOWFUNCTIONS:
- case SHOWPARTITIONS:
- case SHOWLOCKS:
- case SHOWVIEWS:
- case SHOW_ROLES:
- case SHOW_ROLE_PRINCIPALS:
- case SHOW_COMPACTIONS:
- case SHOW_TRANSACTIONS:
- case ABORT_TRANSACTIONS:
- case KILL_QUERY:
- shouldOpenImplicitTxn = false;
- //this implies that no locks are needed for such a command
+ HiveOperation hiveOperation = driverContext.getQueryState().getHiveOperation();
+ switch (hiveOperation == null ? HiveOperation.QUERY : hiveOperation) {
+ case COMMIT:
+ case ROLLBACK:
+ if (!txnManager.isTxnOpen()) {
+ throw new LockException(null, ErrorMsg.OP_NOT_ALLOWED_WITHOUT_TXN, hiveOperation.getOperationName());
+ }
+ case SWITCHDATABASE:
+ case SET_AUTOCOMMIT:
+ /**
+ * autocommit is here for completeness. TM doesn't use it. If we want to support JDBC
+ * semantics (or any other definition of autocommit) it should be done at session level.
+ */
+ case SHOWDATABASES:
+ case SHOWTABLES:
+ case SHOWCOLUMNS:
+ case SHOWFUNCTIONS:
+ case SHOWPARTITIONS:
+ case SHOWLOCKS:
+ case SHOWVIEWS:
+ case SHOW_ROLES:
+ case SHOW_ROLE_PRINCIPALS:
+ case SHOW_COMPACTIONS:
+ case SHOW_TRANSACTIONS:
+ case ABORT_TRANSACTIONS:
+ case KILL_QUERY:
+ shouldOpenImplicitTxn = false;
+ //this implies that no locks are needed for such a command
}
return shouldOpenImplicitTxn;
}
@@ -832,7 +785,7 @@ public class Driver implements IDriver {
throws CommandProcessorException {
if (driverState.isAborted()) {
String errorMessage = "FAILED: command has been interrupted: " + msg;
- console.printError(errorMessage);
+ CONSOLE.printError(errorMessage);
if (hookContext != null) {
try {
invokeFailureHooks(perfLogger, hookContext, errorMessage, null);
@@ -866,18 +819,18 @@ public class Driver implements IDriver {
ASTNode astTree) throws IOException {
String ret = null;
ExplainTask task = new ExplainTask();
- task.initialize(queryState, plan, null, ctx.getOpContext());
+ task.initialize(driverContext.getQueryState(), plan, null, ctx);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintStream ps = new PrintStream(baos);
try {
List<Task<?>> rootTasks = sem.getAllRootTasks();
- if (conf.getBoolVar(ConfVars.HIVE_SERVER2_WEBUI_SHOW_GRAPH)) {
+ if (driverContext.getConf().getBoolVar(ConfVars.HIVE_SERVER2_WEBUI_SHOW_GRAPH)) {
JSONObject jsonPlan = task.getJSONPlan(
null, rootTasks, sem.getFetchTask(), true, true, true, sem.getCboInfo(),
plan.getOptimizedCBOPlan(), plan.getOptimizedQueryString());
if (jsonPlan.getJSONObject(ExplainTask.STAGE_DEPENDENCIES) != null &&
jsonPlan.getJSONObject(ExplainTask.STAGE_DEPENDENCIES).length() <=
- conf.getIntVar(ConfVars.HIVE_SERVER2_WEBUI_MAX_GRAPH_SIZE)) {
+ driverContext.getConf().getIntVar(ConfVars.HIVE_SERVER2_WEBUI_MAX_GRAPH_SIZE)) {
ret = jsonPlan.toString();
} else {
ret = null;
@@ -896,7 +849,7 @@ public class Driver implements IDriver {
@Override
public HiveConf getConf() {
- return conf;
+ return driverContext.getConf();
}
/**
@@ -904,7 +857,7 @@ public class Driver implements IDriver {
*/
@Override
public QueryPlan getPlan() {
- return plan;
+ return driverContext.getPlan();
}
/**
@@ -912,33 +865,33 @@ public class Driver implements IDriver {
*/
@Override
public FetchTask getFetchTask() {
- return fetchTask;
+ return driverContext.getFetchTask();
}
// Write the current set of valid transactions into the conf file
private void recordValidTxns(HiveTxnManager txnMgr) throws LockException {
- String oldTxnString = conf.get(ValidTxnList.VALID_TXNS_KEY);
+ String oldTxnString = driverContext.getConf().get(ValidTxnList.VALID_TXNS_KEY);
if ((oldTxnString != null) && (oldTxnString.length() > 0)) {
throw new IllegalStateException("calling recordValidTxn() more than once in the same " +
JavaUtils.txnIdToString(txnMgr.getCurrentTxnId()));
}
ValidTxnList txnList = txnMgr.getValidTxns();
String txnStr = txnList.toString();
- conf.set(ValidTxnList.VALID_TXNS_KEY, txnStr);
+ driverContext.getConf().set(ValidTxnList.VALID_TXNS_KEY, txnStr);
LOG.debug("Encoding valid txns info " + txnStr + " txnid:" + txnMgr.getCurrentTxnId());
}
// Write the current set of valid write ids for the operated acid tables into the conf file so
// that it can be read by the input format.
private ValidTxnWriteIdList recordValidWriteIds(HiveTxnManager txnMgr) throws LockException {
- String txnString = conf.get(ValidTxnList.VALID_TXNS_KEY);
+ String txnString = driverContext.getConf().get(ValidTxnList.VALID_TXNS_KEY);
if ((txnString == null) || (txnString.isEmpty())) {
throw new IllegalStateException("calling recordValidWritsIdss() without initializing ValidTxnList " +
JavaUtils.txnIdToString(txnMgr.getCurrentTxnId()));
}
- List<String> txnTables = getTransactionalTables(plan);
+ List<String> txnTables = getTransactionalTables(driverContext.getPlan());
ValidTxnWriteIdList txnWriteIds = null;
- if (compactionWriteIds != null) {
+ if (driverContext.getCompactionWriteIds() != null) {
/**
* This is kludgy: here we need to read with Compactor's snapshot/txn
* rather than the snapshot of the current {@code txnMgr}, in effect
@@ -949,27 +902,27 @@ public class Driver implements IDriver {
if (txnTables.size() != 1) {
throw new LockException("Unexpected tables in compaction: " + txnTables);
}
- txnWriteIds = new ValidTxnWriteIdList(compactorTxnId);
- txnWriteIds.addTableValidWriteIdList(compactionWriteIds);
+ txnWriteIds = new ValidTxnWriteIdList(driverContext.getCompactorTxnId());
+ txnWriteIds.addTableValidWriteIdList(driverContext.getCompactionWriteIds());
} else {
txnWriteIds = txnMgr.getValidWriteIds(txnTables, txnString);
}
- if (queryTxnType == TxnType.READ_ONLY && !getWrittenTables(plan).isEmpty()) {
+ if (driverContext.getTxnType() == TxnType.READ_ONLY && !getWrittenTables(driverContext.getPlan()).isEmpty()) {
throw new IllegalStateException(String.format(
"Inferred transaction type '%s' doesn't conform to the actual query string '%s'",
- queryTxnType, queryState.getQueryString()));
+ driverContext.getTxnType(), driverContext.getQueryState().getQueryString()));
}
String writeIdStr = txnWriteIds.toString();
- conf.set(ValidTxnWriteIdList.VALID_TABLES_WRITEIDS_KEY, writeIdStr);
- if (plan.getFetchTask() != null) {
+ driverContext.getConf().set(ValidTxnWriteIdList.VALID_TABLES_WRITEIDS_KEY, writeIdStr);
+ if (driverContext.getPlan().getFetchTask() != null) {
/**
* This is needed for {@link HiveConf.ConfVars.HIVEFETCHTASKCONVERSION} optimization which
* initializes JobConf in FetchOperator before recordValidTxns() but this has to be done
* after locks are acquired to avoid race conditions in ACID.
* This case is supported only for single source query.
*/
- Operator<?> source = plan.getFetchTask().getWork().getSource();
+ Operator<?> source = driverContext.getPlan().getFetchTask().getWork().getSource();
if (source instanceof TableScanOperator) {
TableScanOperator tsOp = (TableScanOperator)source;
String fullTableName = AcidUtils.getFullTableName(tsOp.getConf().getDatabaseName(),
@@ -980,7 +933,7 @@ public class Driver implements IDriver {
+ " is missing from the ValidWriteIdList config: " + writeIdStr);
}
if (writeIdList != null) {
- plan.getFetchTask().setValidWriteIdList(writeIdList.toString());
+ driverContext.getPlan().getFetchTask().setValidWriteIdList(writeIdList.toString());
}
}
}
@@ -1036,10 +989,10 @@ public class Driver implements IDriver {
// Don't use the userName member, as it may or may not have been set. Get the value from
// conf, which calls into getUGI to figure out who the process is running as.
try {
- return conf.getUser();
+ return driverContext.getConf().getUser();
} catch (IOException e) {
String errorMessage = "FAILED: Error in determining user while acquiring locks: " + e.getMessage();
- console.printError(errorMessage, "\n" + StringUtils.stringifyException(e));
+ CONSOLE.printError(errorMessage, "\n" + StringUtils.stringifyException(e));
throw createProcessorException(10, errorMessage, ErrorMsg.findSQLState(e.getMessage()), e);
}
}
@@ -1057,7 +1010,7 @@ public class Driver implements IDriver {
PerfLogger perfLogger = SessionState.getPerfLogger();
perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.ACQUIRE_READ_WRITE_LOCKS);
- if(!queryTxnMgr.isTxnOpen() && queryTxnMgr.supportsAcid()) {
+ if(!driverContext.getTxnManager().isTxnOpen() && driverContext.getTxnManager().supportsAcid()) {
/*non acid txn managers don't support txns but fwd lock requests to lock managers
acid txn manager requires all locks to be associated with a txn so if we
end up here w/o an open txn it's because we are processing something like "use <database>
@@ -1068,8 +1021,8 @@ public class Driver implements IDriver {
String userFromUGI = getUserFromUGI();
// Set the table write id in all of the acid file sinks
- if (!plan.getAcidSinks().isEmpty()) {
- List<FileSinkDesc> acidSinks = new ArrayList<>(plan.getAcidSinks());
+ if (!driverContext.getPlan().getAcidSinks().isEmpty()) {
+ List<FileSinkDesc> acidSinks = new ArrayList<>(driverContext.getPlan().getAcidSinks());
//sorting makes tests easier to write since file names and ROW__IDs depend on statementId
//so this makes (file name -> data) mapping stable
acidSinks.sort((FileSinkDesc fsd1, FileSinkDesc fsd2) ->
@@ -1077,7 +1030,7 @@ public class Driver implements IDriver {
for (FileSinkDesc desc : acidSinks) {
TableDesc tableInfo = desc.getTableInfo();
final TableName tn = HiveTableName.ofNullable(tableInfo.getTableName());
- long writeId = queryTxnMgr.getTableWriteId(tn.getDb(), tn.getTable());
+ long writeId = driverContext.getTxnManager().getTableWriteId(tn.getDb(), tn.getTable());
desc.setTableWriteId(writeId);
/**
@@ -1086,53 +1039,54 @@ public class Driver implements IDriver {
* Insert ... Select ... Union All Select ... using
* {@link org.apache.hadoop.hive.ql.exec.AbstractFileMergeOperator#UNION_SUDBIR_PREFIX}
*/
- desc.setStatementId(queryTxnMgr.getStmtIdAndIncrement());
+ desc.setStatementId(driverContext.getTxnManager().getStmtIdAndIncrement());
String unionAllSubdir = "/" + AbstractFileMergeOperator.UNION_SUDBIR_PREFIX;
if(desc.getInsertOverwrite() && desc.getDirName().toString().contains(unionAllSubdir) &&
desc.isFullAcidTable()) {
- throw new UnsupportedOperationException("QueryId=" + plan.getQueryId() +
+ throw new UnsupportedOperationException("QueryId=" + driverContext.getPlan().getQueryId() +
" is not supported due to OVERWRITE and UNION ALL. Please use truncate + insert");
}
}
}
- if (plan.getAcidAnalyzeTable() != null) {
+ if (driverContext.getPlan().getAcidAnalyzeTable() != null) {
// Allocate write ID for the table being analyzed.
- Table t = plan.getAcidAnalyzeTable().getTable();
- queryTxnMgr.getTableWriteId(t.getDbName(), t.getTableName());
+ Table t = driverContext.getPlan().getAcidAnalyzeTable().getTable();
+ driverContext.getTxnManager().getTableWriteId(t.getDbName(), t.getTableName());
}
- DDLDescWithWriteId acidDdlDesc = plan.getAcidDdlDesc();
+ DDLDescWithWriteId acidDdlDesc = driverContext.getPlan().getAcidDdlDesc();
boolean hasAcidDdl = acidDdlDesc != null && acidDdlDesc.mayNeedWriteId();
if (hasAcidDdl) {
String fqTableName = acidDdlDesc.getFullTableName();
final TableName tn = HiveTableName.ofNullableWithNoDefault(fqTableName);
- long writeId = queryTxnMgr.getTableWriteId(tn.getDb(), tn.getTable());
+ long writeId = driverContext.getTxnManager().getTableWriteId(tn.getDb(), tn.getTable());
acidDdlDesc.setWriteId(writeId);
}
/*It's imperative that {@code acquireLocks()} is called for all commands so that
HiveTxnManager can transition its state machine correctly*/
- queryTxnMgr.acquireLocks(plan, ctx, userFromUGI, driverState);
+ driverContext.getTxnManager().acquireLocks(driverContext.getPlan(), ctx, userFromUGI, driverState);
final List<HiveLock> locks = ctx.getHiveLocks();
- LOG.info("Operation {} obtained {} locks", plan.getOperation(),
+ LOG.info("Operation {} obtained {} locks", driverContext.getPlan().getOperation(),
((locks == null) ? 0 : locks.size()));
// This check is for controlling the correctness of the current state
- if (queryTxnMgr.recordSnapshot(plan) && !validTxnListsGenerated) {
+ if (driverContext.getTxnManager().recordSnapshot(driverContext.getPlan()) &&
+ !driverContext.isValidTxnListsGenerated()) {
throw new IllegalStateException(
"Need to record valid WriteID list but there is no valid TxnID list (" +
- JavaUtils.txnIdToString(queryTxnMgr.getCurrentTxnId()) +
- ", queryId:" + plan.getQueryId() + ")");
+ JavaUtils.txnIdToString(driverContext.getTxnManager().getCurrentTxnId()) +
+ ", queryId:" + driverContext.getPlan().getQueryId() + ")");
}
- if (plan.hasAcidResourcesInQuery() || hasAcidDdl) {
- recordValidWriteIds(queryTxnMgr);
+ if (driverContext.getPlan().hasAcidResourcesInQuery() || hasAcidDdl) {
+ recordValidWriteIds(driverContext.getTxnManager());
}
} catch (Exception e) {
String errorMessage = "FAILED: Error in acquiring locks: " + e.getMessage();
- console.printError(errorMessage, "\n" + StringUtils.stringifyException(e));
+ CONSOLE.printError(errorMessage, "\n" + StringUtils.stringifyException(e));
throw createProcessorException(10, errorMessage, ErrorMsg.findSQLState(e.getMessage()), e);
} finally {
perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.ACQUIRE_READ_WRITE_LOCKS);
@@ -1140,7 +1094,7 @@ public class Driver implements IDriver {
}
public void releaseLocksAndCommitOrRollback(boolean commit) throws LockException {
- releaseLocksAndCommitOrRollback(commit, queryTxnMgr);
+ releaseLocksAndCommitOrRollback(commit, driverContext.getTxnManager());
}
/**
@@ -1156,20 +1110,21 @@ public class Driver implements IDriver {
HiveTxnManager txnMgr;
if (txnManager == null) {
// Default to driver's txn manager if no txn manager specified
- txnMgr = queryTxnMgr;
+ txnMgr = driverContext.getTxnManager();
} else {
txnMgr = txnManager;
}
// If we've opened a transaction we need to commit or rollback rather than explicitly
// releasing the locks.
- conf.unset(ValidTxnList.VALID_TXNS_KEY);
- conf.unset(ValidTxnWriteIdList.VALID_TABLES_WRITEIDS_KEY);
+ driverContext.getConf().unset(ValidTxnList.VALID_TXNS_KEY);
+ driverContext.getConf().unset(ValidTxnWriteIdList.VALID_TABLES_WRITEIDS_KEY);
if(!checkConcurrency()) {
return;
}
if (txnMgr.isTxnOpen()) {
if (commit) {
- if(conf.getBoolVar(ConfVars.HIVE_IN_TEST) && conf.getBoolVar(ConfVars.HIVETESTMODEROLLBACKTXN)) {
+ if (driverContext.getConf().getBoolVar(ConfVars.HIVE_IN_TEST) &&
+ driverContext.getConf().getBoolVar(ConfVars.HIVETESTMODEROLLBACKTXN)) {
txnMgr.rollbackTxn();
}
else {
@@ -1199,7 +1154,7 @@ public class Driver implements IDriver {
*/
public void releaseResources() {
releasePlan();
- releaseDriverContext();
+ releaseTaskQueue();
}
@Override
@@ -1257,7 +1212,7 @@ public class Driver implements IDriver {
StringUtils.stringifyException(cpe.getException()));
}
} catch (HiveException ex) {
- console.printError("Unable to JSON-encode the error", StringUtils.stringifyException(ex));
+ CONSOLE.printError("Unable to JSON-encode the error", StringUtils.stringifyException(ex));
}
throw cpe;
}
@@ -1279,16 +1234,16 @@ public class Driver implements IDriver {
if (cleanupTxnList) {
// Valid txn list might be generated for a query compiled using this
// command, thus we need to reset it
- conf.unset(ValidTxnList.VALID_TXNS_KEY);
+ driverContext.getConf().unset(ValidTxnList.VALID_TXNS_KEY);
}
}
}
public void lockAndRespond() throws CommandProcessorException {
// Assumes the query has already been compiled
- if (plan == null) {
+ if (driverContext.getPlan() == null) {
throw new IllegalStateException(
- "No previously compiled query for driver - queryId=" + queryState.getQueryId());
+ "No previously compiled query for driver - queryId=" + driverContext.getQueryState().getQueryId());
}
if (requiresLock()) {
@@ -1310,7 +1265,7 @@ public class Driver implements IDriver {
PerfLogger perfLogger = SessionState.getPerfLogger(true);
perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.WAIT_COMPILE);
- try (CompileLock compileLock = CompileLockFactory.newInstance(conf, command)) {
+ try (CompileLock compileLock = CompileLockFactory.newInstance(driverContext.getConf(), command)) {
boolean success = compileLock.tryAcquire();
perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.WAIT_COMPILE);
@@ -1337,8 +1292,8 @@ public class Driver implements IDriver {
//Save compile-time PerfLogging for WebUI.
//Execution-time Perf logs are done by either another thread's PerfLogger
//or a reset PerfLogger.
- queryDisplay.setPerfLogStarts(QueryDisplay.Phase.COMPILATION, perfLogger.getStartTimes());
- queryDisplay.setPerfLogEnds(QueryDisplay.Phase.COMPILATION, perfLogger.getEndTimes());
+ driverContext.getQueryDisplay().setPerfLogStarts(QueryDisplay.Phase.COMPILATION, perfLogger.getStartTimes());
+ driverContext.getQueryDisplay().setPerfLogEnds(QueryDisplay.Phase.COMPILATION, perfLogger.getEndTimes());
}
private void runInternal(String command, boolean alreadyCompiled) throws CommandProcessorException {
@@ -1351,7 +1306,7 @@ public class Driver implements IDriver {
driverState.executing();
} else {
String errorMessage = "FAILED: Precompiled query has been cancelled or closed.";
- console.printError(errorMessage);
+ CONSOLE.printError(errorMessage);
throw createProcessorException(12, errorMessage, null, null);
}
} else {
@@ -1365,14 +1320,14 @@ public class Driver implements IDriver {
// the method has been returned by an error or not.
boolean isFinishedWithError = true;
try {
- HiveDriverRunHookContext hookContext = new HiveDriverRunHookContextImpl(conf,
+ HiveDriverRunHookContext hookContext = new HiveDriverRunHookContextImpl(driverContext.getConf(),
alreadyCompiled ? ctx.getCmd() : command);
// Get all the driver run hooks and pre-execute them.
try {
- hookRunner.runPreDriverHooks(hookContext);
+ driverContext.getHookRunner().runPreDriverHooks(hookContext);
} catch (Exception e) {
String errorMessage = "FAILED: Hive Internal Error: " + Utilities.getNameMessage(e);
- console.printError(errorMessage + "\n" + StringUtils.stringifyException(e));
+ CONSOLE.printError(errorMessage + "\n" + StringUtils.stringifyException(e));
throw createProcessorException(12, errorMessage, ErrorMsg.findSQLState(e.getMessage()), e);
}
@@ -1381,7 +1336,7 @@ public class Driver implements IDriver {
compileInternal(command, true);
} else {
// Since we're reusing the compiled plan, we need to update its start time for current run
- plan.setQueryStartTime(queryDisplay.getQueryStartTime());
+ driverContext.getPlan().setQueryStartTime(driverContext.getQueryDisplay().getQueryStartTime());
}
//Reset the PerfLogger so that it doesn't retain any previous values.
@@ -1391,7 +1346,7 @@ public class Driver implements IDriver {
// the reason that we set the txn manager for the cxt here is because each
// query has its own ctx object. The txn mgr is shared across the
// same instance of Driver, which can run multiple queries.
- ctx.setHiveTxnManager(queryTxnMgr);
+ ctx.setHiveTxnManager(driverContext.getTxnManager());
checkInterrupted("at acquiring the lock.", null, null);
@@ -1406,13 +1361,14 @@ public class Driver implements IDriver {
// Currently, we acquire a snapshot, we compile the query wrt that snapshot,
// and then, we acquire locks. If snapshot is still valid, we continue as usual.
// But if snapshot is not valid, we recompile the query.
- retrial = true;
- backupContext.addRewrittenStatementContext(ctx);
- backupContext.setHiveLocks(ctx.getHiveLocks());
- ctx = backupContext;
- conf.set(ValidTxnList.VALID_TXNS_KEY, queryTxnMgr.getValidTxns().toString());
- if (plan.hasAcidResourcesInQuery()) {
- recordValidWriteIds(queryTxnMgr);
+ driverContext.setRetrial(true);
+ driverContext.getBackupContext().addRewrittenStatementContext(ctx);
+ driverContext.getBackupContext().setHiveLocks(ctx.getHiveLocks());
+ ctx = driverContext.getBackupContext();
+ driverContext.getConf().set(ValidTxnList.VALID_TXNS_KEY,
+ driverContext.getTxnManager().getValidTxns().toString());
+ if (driverContext.getPlan().hasAcidResourcesInQuery()) {
+ recordValidWriteIds(driverContext.getTxnManager());
}
if (!alreadyCompiled) {
@@ -1420,7 +1376,7 @@ public class Driver implements IDriver {
compileInternal(command, true);
} else {
// Since we're reusing the compiled plan, we need to update its start time for current run
- plan.setQueryStartTime(queryDisplay.getQueryStartTime());
+ driverContext.getPlan().setQueryStartTime(driverContext.getQueryDisplay().getQueryStartTime());
}
if (!isValidTxnListState()) {
@@ -1434,7 +1390,7 @@ public class Driver implements IDriver {
// the reason that we set the txn manager for the cxt here is because each
// query has its own ctx object. The txn mgr is shared across the
// same instance of Driver, which can run multiple queries.
- ctx.setHiveTxnManager(queryTxnMgr);
+ ctx.setHiveTxnManager(driverContext.getTxnManager());
}
} catch (LockException e) {
throw handleHiveException(e, 13);
@@ -1450,12 +1406,14 @@ public class Driver implements IDriver {
//if needRequireLock is false, the release here will do nothing because there is no lock
try {
//since set autocommit starts an implicit txn, close it
- if(queryTxnMgr.isImplicitTransactionOpen() || plan.getOperation() == HiveOperation.COMMIT) {
+ if (driverContext.getTxnManager().isImplicitTransactionOpen() ||
+ driverContext.getPlan().getOperation() == HiveOperation.COMMIT) {
releaseLocksAndCommitOrRollback(true);
}
- else if(plan.getOperation() == HiveOperation.ROLLBACK) {
+ else if(driverContext.getPlan().getOperation() == HiveOperation.ROLLBACK) {
releaseLocksAndCommitOrRollback(false);
- } else if (!queryTxnMgr.isTxnOpen() && queryState.getHiveOperation() == HiveOperation.REPLLOAD) {
+ } else if (!driverContext.getTxnManager().isTxnOpen() &&
+ driverContext.getQueryState().getHiveOperation() == HiveOperation.REPLLOAD) {
// repl load during migration, commits the explicit txn and start some internal txns. Call
// releaseLocksAndCommitOrRollback to do the clean up.
releaseLocksAndCommitOrRollback(false);
@@ -1466,15 +1424,15 @@ public class Driver implements IDriver {
throw handleHiveException(e, 12);
}
- queryDisplay.setPerfLogStarts(QueryDisplay.Phase.EXECUTION, perfLogger.getStartTimes());
- queryDisplay.setPerfLogEnds(QueryDisplay.Phase.EXECUTION, perfLogger.getEndTimes());
+ driverContext.getQueryDisplay().setPerfLogStarts(QueryDisplay.Phase.EXECUTION, perfLogger.getStartTimes());
+ driverContext.getQueryDisplay().setPerfLogEnds(QueryDisplay.Phase.EXECUTION, perfLogger.getEndTimes());
// Take all the driver run hooks and post-execute them.
try {
- hookRunner.runPostDriverHooks(hookContext);
+ driverContext.getHookRunner().runPostDriverHooks(hookContext);
} catch (Exception e) {
String errorMessage = "FAILED: Hive Internal Error: " + Utilities.getNameMessage(e);
- console.printError(errorMessage + "\n" + StringUtils.stringifyException(e));
+ CONSOLE.printError(errorMessage + "\n" + StringUtils.stringifyException(e));
throw createProcessorException(12, errorMessage, ErrorMsg.findSQLState(e.getMessage()), e);
}
isFinishedWithError = false;
@@ -1482,7 +1440,7 @@ public class Driver implements IDriver {
if (driverState.isAborted()) {
closeInProcess(true);
} else {
- // only release the related resources ctx, driverContext as normal
+ // only release the related resources ctx, taskQueue as normal
releaseResources();
}
@@ -1520,9 +1478,10 @@ public class Driver implements IDriver {
}
String sqlState = e.getCanonicalErrorMsg() != null ?
e.getCanonicalErrorMsg().getSQLState() : ErrorMsg.findSQLState(e.getMessage());
- console.printError(errorMessage + "\n" + StringUtils.stringifyException(e));
+ CONSOLE.printError(errorMessage + "\n" + StringUtils.stringifyException(e));
throw createProcessorException(ret, errorMessage, sqlState, e);
}
+
private boolean requiresLock() {
if (!checkConcurrency()) {
LOG.info("Concurrency mode is disabled, not creating a lock manager");
@@ -1532,24 +1491,24 @@ public class Driver implements IDriver {
if (isExplicitLockOperation()) {
return false;
}
- if (!HiveConf.getBoolVar(conf, ConfVars.HIVE_LOCK_MAPRED_ONLY)) {
+ if (!HiveConf.getBoolVar(driverContext.getConf(), ConfVars.HIVE_LOCK_MAPRED_ONLY)) {
return true;
}
- if (conf.get(Constants.HIVE_QUERY_EXCLUSIVE_LOCK) != null) {
+ if (driverContext.getConf().get(Constants.HIVE_QUERY_EXCLUSIVE_LOCK) != null) {
return true;
}
- Queue<Task<?>> taskQueue = new LinkedList<Task<?>>();
- taskQueue.addAll(plan.getRootTasks());
- while (taskQueue.peek() != null) {
- Task<?> tsk = taskQueue.remove();
+ Queue<Task<?>> tasks = new LinkedList<Task<?>>();
+ tasks.addAll(driverContext.getPlan().getRootTasks());
+ while (tasks.peek() != null) {
+ Task<?> tsk = tasks.remove();
if (tsk.requireLock()) {
return true;
}
if (tsk instanceof ConditionalTask) {
- taskQueue.addAll(((ConditionalTask)tsk).getListTasks());
+ tasks.addAll(((ConditionalTask)tsk).getListTasks());
}
- if (tsk.getChildTasks()!= null) {
- taskQueue.addAll(tsk.getChildTasks());
+ if (tsk.getChildTasks() != null) {
+ tasks.addAll(tsk.getChildTasks());
}
// does not add back up task here, because back up task should be the same
// type of the original task.
@@ -1558,7 +1517,7 @@ public class Driver implements IDriver {
}
private boolean isExplicitLockOperation() {
- HiveOperation currentOpt = plan.getOperation();
+ HiveOperation currentOpt = driverContext.getPlan().getOperation();
if (currentOpt != null) {
switch (currentOpt) {
case LOCKDB:
@@ -1576,7 +1535,7 @@ public class Driver implements IDriver {
private CommandProcessorException createProcessorException(int ret, String errorMessage, String sqlState,
Throwable downstreamError) {
SessionState.getPerfLogger().cleanupPerfLogMetrics();
- queryDisplay.setErrorMessage(errorMessage);
+ driverContext.getQueryDisplay().setErrorMessage(errorMessage);
if (downstreamError != null && downstreamError instanceof HiveException) {
ErrorMsg em = ((HiveException)downstreamError).getCanonicalErrorMsg();
if (em != null) {
@@ -1589,60 +1548,60 @@ public class Driver implements IDriver {
private void useFetchFromCache(CacheEntry cacheEntry) {
// Change query FetchTask to use new location specified in results cache.
FetchTask fetchTaskFromCache = (FetchTask) TaskFactory.get(cacheEntry.getFetchWork());
- fetchTaskFromCache.initialize(queryState, plan, null, ctx.getOpContext());
- plan.setFetchTask(fetchTaskFromCache);
- cacheUsage = new CacheUsage(CacheUsage.CacheStatus.QUERY_USING_CACHE, cacheEntry);
+ fetchTaskFromCache.initialize(driverContext.getQueryState(), driverContext.getPlan(), null, ctx);
+ driverContext.getPlan().setFetchTask(fetchTaskFromCache);
+ driverContext.setCacheUsage(new CacheUsage(CacheUsage.CacheStatus.QUERY_USING_CACHE, cacheEntry));
}
private void preExecutionCacheActions() throws Exception {
- if (cacheUsage != null) {
- if (cacheUsage.getStatus() == CacheUsage.CacheStatus.CAN_CACHE_QUERY_RESULTS &&
- plan.getFetchTask() != null) {
+ if (driverContext.getCacheUsage() != null) {
+ if (driverContext.getCacheUsage().getStatus() == CacheUsage.CacheStatus.CAN_CACHE_QUERY_RESULTS &&
+ driverContext.getPlan().getFetchTask() != null) {
ValidTxnWriteIdList txnWriteIdList = null;
- if (plan.hasAcidResourcesInQuery()) {
- txnWriteIdList = AcidUtils.getValidTxnWriteIdList(conf);
+ if (driverContext.getPlan().hasAcidResourcesInQuery()) {
+ txnWriteIdList = AcidUtils.getValidTxnWriteIdList(driverContext.getConf());
}
// The results of this query execution might be cacheable.
// Add a placeholder entry in the cache so other queries know this result is pending.
CacheEntry pendingCacheEntry =
- QueryResultsCache.getInstance().addToCache(cacheUsage.getQueryInfo(), txnWriteIdList);
+ QueryResultsCache.getInstance().addToCache(driverContext.getCacheUsage().getQueryInfo(), txnWriteIdList);
if (pendingCacheEntry != null) {
// Update cacheUsage to reference the pending entry.
- this.cacheUsage.setCacheEntry(pendingCacheEntry);
+ this.driverContext.getCacheUsage().setCacheEntry(pendingCacheEntry);
}
}
}
}
private void postExecutionCacheActions() throws Exception {
- if (cacheUsage != null) {
- if (cacheUsage.getStatus() == CacheUsage.CacheStatus.QUERY_USING_CACHE) {
+ if (driverContext.getCacheUsage() != null) {
+ if (driverContext.getCacheUsage().getStatus() == CacheUsage.CacheStatus.QUERY_USING_CACHE) {
// Using a previously cached result.
- CacheEntry cacheEntry = cacheUsage.getCacheEntry();
+ CacheEntry cacheEntry = driverContext.getCacheUsage().getCacheEntry();
// Reader count already incremented during cache lookup.
// Save to usedCacheEntry to ensure reader is released after query.
- this.usedCacheEntry = cacheEntry;
- } else if (cacheUsage.getStatus() == CacheUsage.CacheStatus.CAN_CACHE_QUERY_RESULTS &&
- cacheUsage.getCacheEntry() != null &&
- plan.getFetchTask() != null) {
+ driverContext.setUsedCacheEntry(cacheEntry);
+ } else if (driverContext.getCacheUsage().getStatus() == CacheUsage.CacheStatus.CAN_CACHE_QUERY_RESULTS &&
+ driverContext.getCacheUsage().getCacheEntry() != null &&
+ driverContext.getPlan().getFetchTask() != null) {
// Save results to the cache for future queries to use.
PerfLogger perfLogger = SessionState.getPerfLogger();
perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.SAVE_TO_RESULTS_CACHE);
ValidTxnWriteIdList txnWriteIdList = null;
- if (plan.hasAcidResourcesInQuery()) {
- txnWriteIdList = AcidUtils.getValidTxnWriteIdList(conf);
+ if (driverContext.getPlan().hasAcidResourcesInQuery()) {
+ txnWriteIdList = AcidUtils.getValidTxnWriteIdList(driverContext.getConf());
}
- CacheEntry cacheEntry = cacheUsage.getCacheEntry();
+ CacheEntry cacheEntry = driverContext.getCacheUsage().getCacheEntry();
boolean savedToCache = QueryResultsCache.getInstance().setEntryValid(
cacheEntry,
- plan.getFetchTask().getWork());
+ driverContext.getPlan().getFetchTask().getWork());
LOG.info("savedToCache: {} ({})", savedToCache, cacheEntry);
if (savedToCache) {
- useFetchFromCache(cacheUsage.getCacheEntry());
+ useFetchFromCache(driverContext.getCacheUsage().getCacheEntry());
// setEntryValid() already increments the reader count. Set usedCacheEntry so it gets released.
- this.usedCacheEntry = cacheUsage.getCacheEntry();
+ driverContext.setUsedCacheEntry(driverContext.getCacheUsage().getCacheEntry());
}
perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.SAVE_TO_RESULTS_CACHE);
@@ -1654,20 +1613,20 @@ public class Driver implements IDriver {
PerfLogger perfLogger = SessionState.getPerfLogger();
perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.DRIVER_EXECUTE);
- boolean noName = Strings.isNullOrEmpty(conf.get(MRJobConfig.JOB_NAME));
+ boolean noName = Strings.isNullOrEmpty(driverContext.getConf().get(MRJobConfig.JOB_NAME));
int maxlen;
- if ("spark".equals(conf.getVar(ConfVars.HIVE_EXECUTION_ENGINE))) {
- maxlen = conf.getIntVar(HiveConf.ConfVars.HIVESPARKJOBNAMELENGTH);
+ if ("spark".equals(driverContext.getConf().getVar(ConfVars.HIVE_EXECUTION_ENGINE))) {
+ maxlen = driverContext.getConf().getIntVar(HiveConf.ConfVars.HIVESPARKJOBNAMELENGTH);
} else {
- maxlen = conf.getIntVar(HiveConf.ConfVars.HIVEJOBNAMELENGTH);
+ maxlen = driverContext.getConf().getIntVar(HiveConf.ConfVars.HIVEJOBNAMELENGTH);
}
Metrics metrics = MetricsFactory.getInstance();
- String queryId = plan.getQueryId();
+ String queryId = driverContext.getPlan().getQueryId();
// Get the query string from the conf file as the compileInternal() method might
// hide sensitive information during query redaction.
- String queryStr = conf.getQueryString();
+ String queryStr = driverContext.getConf().getQueryString();
driverState.lock();
try {
@@ -1675,7 +1634,7 @@ public class Driver implements IDriver {
// a combined compile/execute in runInternal, throws the error
if (!driverState.isCompiled() && !driverState.isExecuting()) {
String errorMessage = "FAILED: unexpected driverstate: " + driverState + ", for query " + queryStr;
- console.printError(errorMessage);
+ CONSOLE.printError(errorMessage);
throw createProcessorException(1000, errorMessage, "HY008", null);
} else {
driverState.executing();
@@ -1684,8 +1643,6 @@ public class Driver implements IDriver {
driverState.unlock();
}
- maxthreads = HiveConf.getIntVar(conf, HiveConf.ConfVars.EXECPARALLETHREADNUMBER);
-
HookContext hookContext = null;
// Whether there's any error occurred during query execution. Used for query lifetime hook.
@@ -1697,40 +1654,42 @@ public class Driver implements IDriver {
// so clear timing in this thread's Hive object before proceeding.
Hive.get().clearMetaCallTiming();
- plan.setStarted();
+ driverContext.getPlan().setStarted();
if (SessionState.get() != null) {
SessionState.get().getHiveHistory().startQuery(queryStr, queryId);
- SessionState.get().getHiveHistory().logPlanProgress(plan);
+ SessionState.get().getHiveHistory().logPlanProgress(driverContext.getPlan());
}
resStream = null;
SessionState ss = SessionState.get();
// TODO: should this use getUserFromAuthenticator?
- hookContext = new PrivateHookContext(plan, queryState, ctx.getPathToCS(), SessionState.get().getUserName(),
+ hookContext = new PrivateHookContext(driverContext.getPlan(), driverContext.getQueryState(), ctx.getPathToCS(),
+ SessionState.get().getUserName(),
ss.getUserIpAddress(), InetAddress.getLocalHost().getHostAddress(), operationId,
- ss.getSessionId(), Thread.currentThread().getName(), ss.isHiveServerQuery(), perfLogger, queryInfo, ctx);
+ ss.getSessionId(), Thread.currentThread().getName(), ss.isHiveServerQuery(), perfLogger,
+ driverContext.getQueryInfo(), ctx);
hookContext.setHookType(HookContext.HookType.PRE_EXEC_HOOK);
- hookRunner.runPreHooks(hookContext);
+ driverContext.getHookRunner().runPreHooks(hookContext);
// Trigger query hooks before query execution.
- hookRunner.runBeforeExecutionHook(queryStr, hookContext);
+ driverContext.getHookRunner().runBeforeExecutionHook(queryStr, hookContext);
- setQueryDisplays(plan.getRootTasks());
- int mrJobs = Utilities.getMRTasks(plan.getRootTasks()).size();
- int jobs = mrJobs + Utilities.getTezTasks(plan.getRootTasks()).size()
- + Utilities.getSparkTasks(plan.getRootTasks()).size();
+ setQueryDisplays(driverContext.getPlan().getRootTasks());
+ int mrJobs = Utilities.getMRTasks(driverContext.getPlan().getRootTasks()).size();
+ int jobs = mrJobs + Utilities.getTezTasks(driverContext.getPlan().getRootTasks()).size()
+ + Utilities.getSparkTasks(driverContext.getPlan().getRootTasks()).size();
if (jobs > 0) {
logMrWarning(mrJobs);
- console.printInfo("Query ID = " + queryId);
- console.printInfo("Total jobs = " + jobs);
+ CONSOLE.printInfo("Query ID = " + queryId);
+ CONSOLE.printInfo("Total jobs = " + jobs);
}
if (SessionState.get() != null) {
SessionState.get().getHiveHistory().setQueryProperty(queryId, Keys.QUERY_NUM_TASKS,
String.valueOf(jobs));
- SessionState.get().getHiveHistory().setIdToTableMap(plan.getIdToTableNameMap());
+ SessionState.get().getHiveHistory().setIdToTableMap(driverContext.getPlan().getIdToTableNameMap());
}
String jobname = Utilities.abbreviate(queryStr, maxlen - 6);
@@ -1742,22 +1701,21 @@ public class Driver implements IDriver {
checkInterrupted("before running tasks.", hookContext, perfLogger);
- DriverContext driverCxt = new DriverContext(ctx);
- driverCxt.prepare(plan);
+ taskQueue = new TaskQueue(ctx); // for canceling the query (should be bound to session?)
+ taskQueue.prepare(driverContext.getPlan());
ctx.setHDFSCleanup(true);
- this.driverCxt = driverCxt; // for canceling the query (should be bound to session?)
SessionState.get().setMapRedStats(new LinkedHashMap<>());
SessionState.get().setStackTraces(new HashMap<>());
SessionState.get().setLocalMapRedErrors(new HashMap<>());
// Add root Tasks to runnable
- for (Task<?> tsk : plan.getRootTasks()) {
+ for (Task<?> tsk : driverContext.getPlan().getRootTasks()) {
// This should never happen, if it does, it's a bug with the potential to produce
// incorrect results.
assert tsk.getParentTasks() == null || tsk.getParentTasks().isEmpty();
- driverCxt.addToRunnable(tsk);
+ taskQueue.addToRunnable(tsk);
if (metrics != null) {
tsk.updateTaskMetrics(metrics);
@@ -1768,18 +1726,19 @@ public class Driver implements IDriver {
perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.RUN_TASKS);
// Loop while you either have tasks running, or tasks queued up
- while (driverCxt.isRunning()) {
+ while (taskQueue.isRunning()) {
// Launch upto maxthreads tasks
Task<?> task;
- while ((task = driverCxt.getRunnable(maxthreads)) != null) {
- TaskRunner runner = launchTask(task, queryId, noName, jobname, jobs, driverCxt);
+ int maxthreads = HiveConf.getIntVar(driverContext.getConf(), HiveConf.ConfVars.EXECPARALLETHREADNUMBER);
+ while ((task = taskQueue.getRunnable(maxthreads)) != null) {
+ TaskRunner runner = launchTask(task, queryId, noName, jobname, jobs, taskQueue);
if (!runner.isRunning()) {
break;
}
}
// poll the Tasks to see which one completed
- TaskRunner tskRun = driverCxt.pollFinished();
+ TaskRunner tskRun = taskQueue.pollFinished();
if (tskRun == null) {
continue;
}
@@ -1789,14 +1748,14 @@ public class Driver implements IDriver {
Briefly for replication the graph is huge and so memory pressure is going to be huge if
we keep a lot of references around.
*/
- String opName = plan.getOperationName();
+ String opName = driverContext.getPlan().getOperationName();
boolean isReplicationOperation = opName.equals(HiveOperation.REPLDUMP.getOperationName())
|| opName.equals(HiveOperation.REPLLOAD.getOperationName());
if (!isReplicationOperation) {
hookContext.addCompleteTask(tskRun);
}
- queryDisplay.setTaskResult(tskRun.getTask().getId(), tskRun.getTaskResult());
+ driverContext.getQueryDisplay().setTaskResult(tskRun.getTask().getId(), tskRun.getTaskResult());
Task<?> tsk = tskRun.getTask();
TaskResult result = tskRun.getTaskResult();
@@ -1808,19 +1767,19 @@ public class Driver implements IDriver {
Task<?> backupTask = tsk.getAndInitBackupTask();
if (backupTask != null) {
String errorMessage = getErrorMsgAndDetail(exitVal, result.getTaskError(), tsk);
- console.printError(errorMessage);
+ CONSOLE.printError(errorMessage);
errorMessage = "ATTEMPT: Execute BackupTask: " + backupTask.getClass().getName();
- console.printError(errorMessage);
+ CONSOLE.printError(errorMessage);
// add backup task to runnable
- if (DriverContext.isLaunchable(backupTask)) {
- driverCxt.addToRunnable(backupTask);
+ if (TaskQueue.isLaunchable(backupTask)) {
+ taskQueue.addToRunnable(backupTask);
}
continue;
} else {
String errorMessage = getErrorMsgAndDetail(exitVal, result.getTaskError(), tsk);
- if (driverCxt.isShutdown()) {
+ if (taskQueue.isShutdown()) {
errorMessage = "FAILED: Operation cancelled. " + errorMessage;
}
invokeFailureHooks(perfLogger, hookContext,
@@ -1837,8 +1796,8 @@ public class Driver implements IDriver {
}
}
- console.printError(errorMessage);
- driverCxt.shutdown();
+ CONSOLE.printError(errorMessage);
+ taskQueue.shutdown();
// in case we decided to run everything in local mode, restore the
// the jobtracker setting to its initial value
ctx.restoreOriginalTracker();
@@ -1846,7 +1805,7 @@ public class Driver implements IDriver {
}
}
- driverCxt.finished(tskRun);
+ taskQueue.finished(tskRun);
if (SessionState.get() != null) {
SessionState.get().getHiveHistory().setTaskProperty(queryId, tsk.getId(),
@@ -1856,8 +1815,8 @@ public class Driver implements IDriver {
if (tsk.getChildTasks() != null) {
for (Task<?> child : tsk.getChildTasks()) {
- if (DriverContext.isLaunchable(child)) {
- driverCxt.addToRunnable(child);
+ if (TaskQueue.isLaunchable(child)) {
+ taskQueue.addToRunnable(child);
}
}
}
@@ -1870,10 +1829,10 @@ public class Driver implements IDriver {
// the jobtracker setting to its initial value
ctx.restoreOriginalTracker();
- if (driverCxt.isShutdown()) {
+ if (taskQueue.isShutdown()) {
String errorMessage = "FAILED: Operation cancelled";
invokeFailureHooks(perfLogger, hookContext, errorMessage, null);
- console.printError(errorMessage);
+ CONSOLE.printError(errorMessage);
throw createProcessorException(1000, errorMessage, "HY008", null);
}
@@ -1881,27 +1840,27 @@ public class Driver implements IDriver {
// Some incomplete outputs may be added at the beginning, for eg: for dynamic partitions.
// remove them
HashSet<WriteEntity> remOutputs = new LinkedHashSet<WriteEntity>();
- for (WriteEntity output : plan.getOutputs()) {
+ for (WriteEntity output : driverContext.getPlan().getOutputs()) {
if (!output.isComplete()) {
remOutputs.add(output);
}
}
for (WriteEntity output : remOutputs) {
- plan.getOutputs().remove(output);
+ driverContext.getPlan().getOutputs().remove(output);
}
hookContext.setHookType(HookContext.HookType.POST_EXEC_HOOK);
- hookRunner.runPostExecHooks(hookContext);
+ driverContext.getHookRunner().runPostExecHooks(hookContext);
if (SessionState.get() != null) {
SessionState.get().getHiveHistory().setQueryProperty(queryId, Keys.QUERY_RET_CODE,
String.valueOf(0));
SessionState.get().getHiveHistory().printRowCount(queryId);
}
- releasePlan(plan);
+ releasePlan(driverContext.getPlan());
} catch (CommandProcessorException cpe) {
executionError = true;
throw cpe;
@@ -1924,12 +1883,12 @@ public class Driver implements IDriver {
LOG.warn("Failed to invoke failure hook", t);
}
}
- console.printError(errorMessage + "\n" + StringUtils.stringifyException(e));
+ CONSOLE.printError(errorMessage + "\n" + StringUtils.stringifyException(e));
throw createProcessorException(12, errorMessage, "08S01", e);
} finally {
// Trigger query hooks after query completes its execution.
try {
- hookRunner.runAfterExecutionHook(queryStr, hookContext, executionError);
+ driverContext.getHookRunner().runAfterExecutionHook(queryStr, hookContext, executionError);
} catch (Exception e) {
LOG.warn("Failed when invoking query after execution hook", e);
}
@@ -1938,20 +1897,20 @@ public class Driver implements IDriver {
SessionState.get().getHiveHistory().endQuery(queryId);
}
if (noName) {
- conf.set(MRJobConfig.JOB_NAME, "");
+ driverContext.getConf().set(MRJobConfig.JOB_NAME, "");
}
double duration = perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.DRIVER_EXECUTE)/1000.00;
ImmutableMap<String, Long> executionHMSTimings = dumpMetaCallTimingWithoutEx("execution");
- queryDisplay.setHmsTimings(QueryDisplay.Phase.EXECUTION, executionHMSTimings);
+ driverContext.getQueryDisplay().setHmsTimings(QueryDisplay.Phase.EXECUTION, executionHMSTimings);
Map<String, MapRedStats> stats = SessionState.get().getMapRedStats();
if (stats != null && !stats.isEmpty()) {
long totalCpu = 0;
long numModifiedRows = 0;
- console.printInfo("MapReduce Jobs Launched: ");
+ CONSOLE.printInfo("MapReduce Jobs Launched: ");
for (Map.Entry<String, MapRedStats> entry : stats.entrySet()) {
- console.printInfo("Stage-" + entry.getKey() + ": " + entry.getValue());
+ CONSOLE.printInfo("Stage-" + entry.getKey() + ": " + entry.getValue());
totalCpu += entry.getValue().getCpuMSec();
if (numModifiedRows > -1) {
@@ -1959,8 +1918,8 @@ public class Driver implements IDriver {
numModifiedRows = addWithOverflowCheck(numModifiedRows, entry.getValue().getNumModifiedRows());
}
}
- queryState.setNumModifiedRows(numModifiedRows);
- console.printInfo("Total MapReduce CPU Time Spent: " + Utilities.formatMsecToStr(totalCpu));
+ driverContext.getQueryState().setNumModifiedRows(numModifiedRows);
+ CONSOLE.printInfo("Total MapReduce CPU Time Spent: " + Utilities.formatMsecToStr(totalCpu));
}
SparkSession ss = SessionState.get().getSparkSession();
if (ss != null) {
@@ -2025,7 +1984,7 @@ public class Driver implements IDriver {
if (visited.contains(task)) {
continue;
}
- task.setQueryDisplay(queryDisplay);
+ task.setQueryDisplay(driverContext.getQueryDisplay());
if (task.getDependentTasks() != null) {
childTasks.addAll(task.getDependentTasks());
}
@@ -2035,7 +1994,7 @@ public class Driver implements IDriver {
}
private void logMrWarning(int mrJobs) {
- if (mrJobs <= 0 || !("mr".equals(HiveConf.getVar(conf, ConfVars.HIVE_EXECUTION_ENGINE)))) {
+ if (mrJobs <= 0 || !("mr".equals(HiveConf.getVar(driverContext.getConf(), ConfVars.HIVE_EXECUTION_ENGINE)))) {
return;
}
String warning = HiveConf.generateMrDeprecationWarning();
@@ -2069,7 +2028,7 @@ public class Driver implements IDriver {
hookContext.setErrorMessage(errorMessage);
hookContext.setException(exception);
// Get all the failure execution hooks and execute them.
- hookRunner.runFailureHooks(hookContext);
+ driverContext.getHookRunner().runFailureHooks(hookContext);
}
/**
@@ -2085,27 +2044,27 @@ public class Driver implements IDriver {
* name of the task, if it is a map-reduce job
* @param jobs
* number of map-reduce jobs
- * @param cxt
- * the driver context
+ * @param taskQueue
+ * the task queue
*/
private TaskRunner launchTask(Task<?> tsk, String queryId, boolean noName,
- String jobname, int jobs, DriverContext cxt) throws HiveException {
+ String jobname, int jobs, TaskQueue taskQueue) throws HiveException {
if (SessionState.get() != null) {
SessionState.get().getHiveHistory().startTask(queryId, tsk, tsk.getClass().getName());
}
if (tsk.isMapRedTask() && !(tsk instanceof ConditionalTask)) {
if (noName) {
- conf.set(MRJobConfig.JOB_NAME, jobname + " (" + tsk.getId() + ")");
+ driverContext.getConf().set(MRJobConfig.JOB_NAME, jobname + " (" + tsk.getId() + ")");
}
- conf.set(DagUtils.MAPREDUCE_WORKFLOW_NODE_NAME, tsk.getId());
- Utilities.setWorkflowAdjacencies(conf, plan);
- cxt.incCurJobNo(1);
- console.printInfo("Launching Job " + cxt.getCurJobNo() + " out of " + jobs);
+ driverContext.getConf().set(DagUtils.MAPREDUCE_WORKFLOW_NODE_NAME, tsk.getId());
+ Utilities.setWorkflowAdjacencies(driverContext.getConf(), driverContext.getPlan());
+ taskQueue.incCurJobNo(1);
+ CONSOLE.printInfo("Launching Job " + taskQueue.getCurJobNo() + " out of " + jobs);
}
- tsk.initialize(queryState, plan, cxt, ctx.getOpContext());
- TaskRunner tskRun = new TaskRunner(tsk, cxt);
+ tsk.initialize(driverContext.getQueryState(), driverContext.getPlan(), taskQueue, ctx);
+ TaskRunner tskRun = new TaskRunner(tsk, taskQueue);
- cxt.launching(tskRun);
+ taskQueue.launching(tskRun);
// Launch Task
if (HiveConf.getBoolVar(tsk.getConf(), HiveConf.ConfVars.EXECPARALLEL) && tsk.canExecuteInParallel()) {
// Launch it in the parallel mode, as a separate thread only for MR tasks
@@ -2124,7 +2083,7 @@ public class Driver implements IDriver {
@Override
public boolean isFetchingTable() {
- return fetchTask != null;
+ return driverContext.getFetchTask() != null;
}
@SuppressWarnings("unchecked")
@@ -2140,11 +2099,11 @@ public class Driver implements IDriver {
* indeed written using ThriftJDBCBinarySerDe, read one row from the output sequence file,
* since it is a blob of row batches.
*/
- if (fetchTask.getWork().isUsingThriftJDBCBinarySerDe()) {
+ if (driverContext.getFetchTask().getWork().isUsingThriftJDBCBinarySerDe()) {
maxRows = 1;
}
- fetchTask.setMaxRows(maxRows);
- return fetchTask.fetch(res);
+ driverContext.getFetchTask().setMaxRows(maxRows);
+ return driverContext.getFetchTask().fetch(res);
}
if (resStream == null) {
@@ -2182,7 +2141,7 @@ public class Driver implements IDriver {
}
row = null;
} catch (IOException e) {
- console.printError("FAILED: Unexpected IO exception : " + e.getMessage());
+ CONSOLE.printError("FAILED: Unexpected IO exception : " + e.getMessage());
return false;
}
@@ -2200,26 +2159,26 @@ public class Driver implements IDriver {
}
if (isFetchingTable()) {
try {
- fetchTask.clearFetch();
+ driverContext.getFetchTask().clearFetch();
} catch (Exception e) {
throw new IOException("Error closing the current fetch task", e);
}
// FetchTask should not depend on the plan.
- fetchTask.initialize(queryState, null, null, ctx.getOpContext());
+ driverContext.getFetchTask().initialize(driverContext.getQueryState(), null, null, ctx);
} else {
ctx.resetStream();
resStream = null;
}
}
- // DriverContext could be released in the query and close processes at same
+ // TaskQueue could be released in the query and close processes at same
// time, which needs to be thread protected.
- private void releaseDriverContext() {
+ private void releaseTaskQueue() {
driverState.lock();
try {
- if (driverCxt != null) {
- driverCxt.shutdown();
- driverCxt = null;
+ if (taskQueue != null) {
+ taskQueue.shutdown();
+ taskQueue = null;
}
} catch (Exception e) {
LOG.debug("Exception while shutting down the task runner", e);
@@ -2230,14 +2189,15 @@ public class Driver implements IDriver {
private void releasePlan() {
try {
- if (plan != null) {
- fetchTask = plan.getFetchTask();
+ if (driverContext.getPlan() != null) {
+ FetchTask fetchTask = driverContext.getPlan().getFetchTask();
if (fetchTask != null) {
- fetchTask.setDriverContext(null);
+ fetchTask.setTaskQueue(null);
fetchTask.setQueryPlan(null);
}
+ driverContext.setFetchTask(fetchTask);
}
- plan = null;
+ driverContext.setPlan(null);
} catch (Exception e) {
LOG.debug("Exception while clearing the Fetch task", e);
}
@@ -2248,8 +2208,8 @@ public class Driver implements IDriver {
if (ctx != null) {
boolean deleteResultDir = true;
// don't let context delete result dirs and scratch dirs if result was cached
- if(this.cacheUsage != null
- && this.cacheUsage.getStatus() == CacheUsage.CacheStatus.QUERY_USING_CACHE) {
+ if (driverContext.getCacheUsage() != null
+ && driverContext.getCacheUsage().getStatus() == CacheUsage.CacheStatus.QUERY_USING_CACHE) {
deleteResultDir = false;
}
@@ -2278,9 +2238,9 @@ public class Driver implements IDriver {
private void releaseFetchTask() {
try {
- if (fetchTask != null) {
- fetchTask.clearFetch();
- fetchTask = null;
+ if (driverContext.getFetchTask() != null) {
+ driverContext.getFetchTask().clearFetch();
+ driverContext.setFetchTask(null);
}
} catch (Exception e) {
LOG.debug(" Exception while clearing the FetchTask ", e);
@@ -2290,35 +2250,35 @@ public class Driver implements IDriver {
private boolean hasBadCacheAttempt() {
// Check if the query results were cacheable, and created a pending cache entry.
// If we successfully saved the results, the usage would have changed to QUERY_USING_CACHE.
- return (cacheUsage != null &&
- cacheUsage.getStatus() == CacheUsage.CacheStatus.CAN_CACHE_QUERY_RESULTS &&
- cacheUsage.getCacheEntry() != null);
+ return (driverContext.getCacheUsage() != null &&
+ driverContext.getCacheUsage().getStatus() == CacheUsage.CacheStatus.CAN_CACHE_QUERY_RESULTS &&
+ driverContext.getCacheUsage().getCacheEntry() != null);
}
private void releaseCachedResult() {
// Assumes the reader count has been incremented automatically by the results cache by either
// lookup or creating the cache entry.
- if (usedCacheEntry != null) {
- usedCacheEntry.releaseReader();
- usedCacheEntry = null;
+ if (driverContext.getUsedCacheEntry() != null) {
+ driverContext.getUsedCacheEntry().releaseReader();
+ driverContext.setUsedCacheEntry(null);
} else if (hasBadCacheAttempt()) {
// This query create a pending cache entry but it was never saved with real results, cleanup.
// This step is required, as there may be queries waiting on this pending cache entry.
// Removing/invalidating this entry will notify the waiters that this entry cannot be used.
try {
- QueryResultsCache.getInstance().removeEntry(cacheUsage.getCacheEntry());
+ QueryResultsCache.getInstance().removeEntry(driverContext.getCacheUsage().getCacheEntry());
} catch (Exception err) {
- LOG.error("Error removing failed cache entry " + cacheUsage.getCacheEntry(), err);
+ LOG.error("Error removing failed cache entry " + driverContext.getCacheUsage().getCacheEntry(), err);
}
}
- cacheUsage = null;
+ driverContext.setCacheUsage(null);
}
// Close and release resources within a running query process. Since it runs under
// driver state COMPILING, EXECUTING or INTERRUPT, it would not have race condition
// with the releases probably running in the other closing thread.
private int closeInProcess(boolean destroyed) {
- releaseDriverContext();
+ releaseTaskQueue();
releasePlan();
releaseCachedResult();
releaseFetchTask();
@@ -2343,7 +2303,7 @@ public class Driver implements IDriver {
public void close() {
driverState.lock();
try {
- releaseDriverContext();
+ releaseTaskQueue();
if (driverState.isCompiling() || driverState.isExecuting()) {
driverState.abort();
}
@@ -2389,7 +2349,7 @@ public class Driver implements IDriver {
@Override
public QueryDisplay getQueryDisplay() {
- return queryDisplay;
+ return driverContext.getQueryDisplay();
}
/**
@@ -2403,36 +2363,37 @@ public class Driver implements IDriver {
@Override
public QueryState getQueryState() {
- return queryState;
+ return driverContext.getQueryState();
}
public HookRunner getHookRunner() {
- return hookRunner;
+ return driverContext.getHookRunner();
}
public void setStatsSource(StatsSource runtimeStatsSource) {
- this.statsSource = runtimeStatsSource;
+ driverContext.setStatsSource(runtimeStatsSource);
}
public StatsSource getStatsSource() {
- return statsSource;
+ return driverContext.getStatsSource();
}
@Override
public boolean hasResultSet() {
// TODO explain should use a FetchTask for reading
- for (Task<?> task : plan.getRootTasks()) {
+ for (Task<?> task : driverContext.getPlan().getRootTasks()) {
if (task.getClass() == ExplainTask.class) {
return true;
}
}
- return plan.getFetchTask() != null && plan.getResultSchema() != null && plan.getResultSchema().isSetFieldSchemas();
+ return driverContext.getPlan().getFetchTask() != null && driverContext.getPlan().getResultSchema() != null &&
+ driverContext.getPlan().getResultSchema().isSetFieldSchemas();
}
- void setCompactionWriteIds(ValidWriteIdList val, long compactorTxnId) {
- this.compactionWriteIds = val;
- this.compactorTxnId = compactorTxnId;
+ void setCompactionWriteIds(ValidWriteIdList compactionWriteIds, long compactorTxnId) {
+ driverContext.setCompactionWriteIds(compactionWriteIds);
+ driverContext.setCompactorTxnId(compactorTxnId);
}
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/DriverContext.java b/ql/src/java/org/apache/hadoop/hive/ql/DriverContext.java
index 1b8260a..1afcfc8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/DriverContext.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/DriverContext.java
@@ -18,220 +18,199 @@
package org.apache.hadoop.hive.ql;
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Queue;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-
-import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
-import org.apache.hadoop.hive.ql.exec.NodeUtils;
-import org.apache.hadoop.hive.ql.exec.NodeUtils.Function;
-import org.apache.hadoop.hive.ql.exec.Operator;
-import org.apache.hadoop.hive.ql.exec.StatsTask;
-import org.apache.hadoop.hive.ql.exec.Task;
-import org.apache.hadoop.hive.ql.exec.TaskRunner;
-import org.apache.hadoop.hive.ql.exec.mr.MapRedTask;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.plan.MapWork;
-import org.apache.hadoop.hive.ql.plan.ReduceWork;
-import org.apache.hadoop.hive.ql.session.SessionState;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.hadoop.hive.common.ValidWriteIdList;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.Schema;
+import org.apache.hadoop.hive.metastore.api.TxnType;
+import org.apache.hadoop.hive.ql.cache.results.CacheUsage;
+import org.apache.hadoop.hive.ql.cache.results.QueryResultsCache.CacheEntry;
+import org.apache.hadoop.hive.ql.exec.FetchTask;
+import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager;
+import org.apache.hadoop.hive.ql.plan.mapper.StatsSource;
/**
- * DriverContext.
- *
+ * Context for the procedure managed by the Driver.
*/
public class DriverContext {
+ // For WebUI. Kept alive after queryPlan is freed.
+ private final QueryDisplay queryDisplay = new QueryDisplay();
+
+ private final QueryState queryState;
+ private final QueryInfo queryInfo;
+ private final HiveConf conf;
+ private final String userName;
+ private final HookRunner hookRunner;
+
+ // Transaction manager the Driver has been initialized with (can be null).
+ // If this is set then this Transaction manager will be used during query
+ // compilation/execution rather than using the current session's transaction manager.
+ // This might be needed in a situation where a Driver is nested within an already
+ // running Driver/query - the nested Driver requires a separate transaction manager
+ // so as not to conflict with the outer Driver/query which is using the session
+ // transaction manager.
+ private final HiveTxnManager initTxnManager;
+
+ private QueryPlan plan;
+ private Schema schema;
+
+ private FetchTask fetchTask;
+ // Transaction manager used for the query. This will be set at compile time based on
+ // either initTxnMgr or from the SessionState, in that order.
+ private HiveTxnManager txnManager;
+ private TxnType txnType = TxnType.DEFAULT;
+ private StatsSource statsSource;
+
+ // Boolean to store information about whether valid txn list was generated
+ // for current query.
+ private boolean validTxnListsGenerated;
+
+ private CacheUsage cacheUsage;
+ private CacheEntry usedCacheEntry;
+ private ValidWriteIdList compactionWriteIds = null;
+ private long compactorTxnId = 0;
+
+ private Context backupContext = null;
+ private boolean retrial = false;
+
+ public DriverContext(QueryState queryState, QueryInfo queryInfo, String userName, HookRunner hookRunner,
+ HiveTxnManager initTxnManager) {
+ this.queryState = queryState;
+ this.queryInfo = queryInfo;
+ this.conf = queryState.getConf();
+ this.userName = userName;
+ this.hookRunner = hookRunner;
+ this.initTxnManager = initTxnManager;
+ }
+
+ public QueryDisplay getQueryDisplay() {
+ return queryDisplay;
+ }
+
+ public QueryState getQueryState() {
+ return queryState;
+ }
+
+ public QueryInfo getQueryInfo() {
+ return queryInfo;
+ }
+
+ public HiveConf getConf() {
+ return conf;
+ }
+
+ public String getUserName() {
+ return userName;
+ }
+
+ public HookRunner getHookRunner() {
+ return hookRunner;
+ }
+
+ public HiveTxnManager getInitTxnManager() {
+ return initTxnManager;
+ }
+
+ public QueryPlan getPlan() {
+ return plan;
+ }
- private static final Logger LOG = LoggerFactory.getLogger(Driver.class.getName());
- private static final SessionState.LogHelper console = new SessionState.LogHelper(LOG);
+ public void setPlan(QueryPlan plan) {
+ this.plan = plan;
+ }
- private static final int SLEEP_TIME = 2000;
+ public Schema getSchema() {
+ return schema;
+ }
- private Queue<Task<?>> runnable;
- private Queue<TaskRunner> running;
+ public void setSchema(Schema schema) {
+ this.schema = schema;
+ }
- // how many jobs have been started
- private int curJobNo;
+ public FetchTask getFetchTask() {
+ return fetchTask;
+ }
- private Context ctx;
- private boolean shutdown;
+ public void setFetchTask(FetchTask fetchTask) {
+ this.fetchTask = fetchTask;
+ }
- final Map<String, StatsTask> statsTasks = new HashMap<>(1);
+ public HiveTxnManager getTxnManager() {
+ return txnManager;
+ }
+
+ public void setTxnManager(HiveTxnManager txnManager) {
+ this.txnManager = txnManager;
+ }
+
+ public TxnType getTxnType() {
+ return txnType;
+ }
+
+ public void setTxnType(TxnType txnType) {
+ this.txnType = txnType;
+ }
+
+ public StatsSource getStatsSource() {
+ return statsSource;
+ }
+
+ public void setStatsSource(StatsSource statsSource) {
+ this.statsSource = statsSource;
+ }
+
+ public boolean isValidTxnListsGenerated() {
+ return validTxnListsGenerated;
+ }
+
+ public void setValidTxnListsGenerated(boolean validTxnListsGenerated) {
+ this.validTxnListsGenerated = validTxnListsGenerated;
+ }
+
+ public CacheUsage getCacheUsage() {
+ return cacheUsage;
+ }
+
+ public void setCacheUsage(CacheUsage cacheUsage) {
+ this.cacheUsage = cacheUsage;
+ }
- public DriverContext() {
+ public CacheEntry getUsedCacheEntry() {
+ return usedCacheEntry;
}
- public DriverContext(Context ctx) {
- this.runnable = new ConcurrentLinkedQueue<Task<?>>();
- this.running = new LinkedBlockingQueue<TaskRunner>();
- this.ctx = ctx;
+ public void setUsedCacheEntry(CacheEntry usedCacheEntry) {
+ this.usedCacheEntry = usedCacheEntry;
}
- public synchronized boolean isShutdown() {
- return shutdown;
+ public ValidWriteIdList getCompactionWriteIds() {
+ return compactionWriteIds;
}
- public synchronized boolean isRunning() {
- return !shutdown && (!running.isEmpty() || !runnable.isEmpty());
+ public void setCompactionWriteIds(ValidWriteIdList compactionWriteIds) {
+ this.compactionWriteIds = compactionWriteIds;
}
- public synchronized void remove(Task<?> task) {
- runnable.remove(task);
+ public long getCompactorTxnId() {
+ return compactorTxnId;
}
- public synchronized void launching(TaskRunner runner) throws HiveException {
- checkShutdown();
- running.add(runner);
+ public void setCompactorTxnId(long compactorTxnId) {
+ this.compactorTxnId = compactorTxnId;
}
- public synchronized Task<?> getRunnable(int maxthreads) throws HiveException {
- checkShutdown();
- if (runnable.peek() != null && running.size() < maxthreads) {
- return runnable.remove();
- }
- return null;
+ public Context getBackupContext() {
+ return backupContext;
}
- public synchronized void releaseRunnable() {
- //release the waiting poller.
- notify();
+ public void setBackupContext(Context backupContext) {
+ this.backupContext = backupContext;
}
- /**
- * Polls running tasks to see if a task has ended.
- *
- * @return The result object for any completed/failed task
- */
- public synchronized TaskRunner pollFinished() throws InterruptedException {
- while (!shutdown) {
- Iterator<TaskRunner> it = running.iterator();
- while (it.hasNext()) {
- TaskRunner runner = it.next();
- if (runner != null && !runner.isRunning()) {
- it.remove();
- return runner;
- }
- }
- wait(SLEEP_TIME);
- }
- return null;
+ public boolean isRetrial() {
+ return retrial;
}
- private void checkShutdown() throws HiveException {
- if (shutdown) {
- throw new HiveException("FAILED: Operation cancelled");
- }
- }
- /**
- * Cleans up remaining tasks in case of failure
- */
- public synchronized void shutdown() {
- LOG.debug("Shutting down query " + ctx.getCmd());
- shutdown = true;
- for (TaskRunner runner : running) {
- if (runner.isRunning()) {
- Task<?> task = runner.getTask();
- LOG.warn("Shutting down task : " + task);
- try {
- task.shutdown();
- } catch (Exception e) {
- console.printError("Exception on shutting down task " + task.getId() + ": " + e);
- }
- Thread thread = runner.getRunner();
- if (thread != null) {
- thread.interrupt();
- }
- }
- }
- running.clear();
- }
-
- /**
- * Checks if a task can be launched.
- *
- * @param tsk
- * the task to be checked
- * @return true if the task is launchable, false otherwise
- */
-
- public static boolean isLaunchable(Task<?> tsk) {
- // A launchable task is one that hasn't been queued, hasn't been
- // initialized, and is runnable.
- return tsk.isNotInitialized() && tsk.isRunnable();
- }
-
- public synchronized boolean addToRunnable(Task<?> tsk) throws HiveException {
- if (runnable.contains(tsk)) {
- return false;
- }
- checkShutdown();
- runnable.add(tsk);
- tsk.setQueued();
- return true;
- }
-
- public int getCurJobNo() {
- return curJobNo;
- }
-
- public Context getCtx() {
- return ctx;
- }
-
- public void incCurJobNo(int amount) {
- this.curJobNo = this.curJobNo + amount;
- }
-
- public void prepare(QueryPlan plan) {
- // extract stats keys from StatsTask
- List<Task<?>> rootTasks = plan.getRootTasks();
- NodeUtils.iterateTask(rootTasks, StatsTask.class, new Function<StatsTask>() {
- @Override
- public void apply(StatsTask statsTask) {
- if (statsTask.getWork().isAggregating()) {
- statsTasks.put(statsTask.getWork().getAggKey(), statsTask);
- }
- }
- });
- }
-
- public void prepare(TaskRunner runner) {
- }
-
- public void finished(TaskRunner runner) {
- if (statsTasks.isEmpty() || !(runner.getTask() instanceof MapRedTask)) {
- return;
- }
- MapRedTask mapredTask = (MapRedTask) runner.getTask();
-
- MapWork mapWork = mapredTask.getWork().getMapWork();
- ReduceWork reduceWork = mapredTask.getWork().getReduceWork();
- List<Operator> operators = new ArrayList<Operator>(mapWork.getAliasToWork().values());
- if (reduceWork != null) {
- operators.add(reduceWork.getReducer());
- }
- final List<String> statKeys = new ArrayList<String>(1);
- NodeUtils.iterate(operators, FileSinkOperator.class, new Function<FileSinkOperator>() {
- @Override
- public void apply(FileSinkOperator fsOp) {
- if (fsOp.getConf().isGatherStats()) {
- statKeys.add(fsOp.getConf().getStatsAggPrefix());
- }
- }
- });
- for (String statKey : statKeys) {
- if (statsTasks.containsKey(statKey)) {
- statsTasks.get(statKey).getWork().setSourceTask(mapredTask);
- } else {
- LOG.debug("There is no correspoing statTask for: " + statKey);
- }
- }
+ public void setRetrial(boolean retrial) {
+ this.retrial = retrial;
}
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/DriverUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/DriverUtils.java
index 596b109..26e904a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/DriverUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/DriverUtils.java
@@ -54,8 +54,8 @@ public class DriverUtils {
try {
driver.run(query);
} catch (CommandProcessorException e) {
- LOG.error("Failed to run " + query, e.getException());
- throw new HiveException("Failed to run " + query, e.getException());
+ LOG.error("Failed to run " + query, e);
+ throw new HiveException("Failed to run " + query, e);
}
} finally {
driver.close();
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java b/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java
index 07fe641..9c61b31 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java
@@ -267,7 +267,6 @@ public class QueryProperties {
/**
* True indicates this statement create or replaces a materialized view, not that it is a query
* against a materialized view.
- * @return
*/
public boolean isMaterializedView() {
return isMaterializedView;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/DriverContext.java b/ql/src/java/org/apache/hadoop/hive/ql/TaskQueue.java
similarity index 89%
copy from ql/src/java/org/apache/hadoop/hive/ql/DriverContext.java
copy to ql/src/java/org/apache/hadoop/hive/ql/TaskQueue.java
index 1b8260a..b585982 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/DriverContext.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/TaskQueue.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hive.ql;
-import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
@@ -26,7 +25,6 @@ import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.LinkedBlockingQueue;
import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
import org.apache.hadoop.hive.ql.exec.NodeUtils;
@@ -44,33 +42,29 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * DriverContext.
- *
+ * Handles the queue of tasks that should be executed by the driver.
*/
-public class DriverContext {
+public class TaskQueue {
private static final Logger LOG = LoggerFactory.getLogger(Driver.class.getName());
- private static final SessionState.LogHelper console = new SessionState.LogHelper(LOG);
+ private static final SessionState.LogHelper CONSOLE = new SessionState.LogHelper(LOG);
private static final int SLEEP_TIME = 2000;
- private Queue<Task<?>> runnable;
- private Queue<TaskRunner> running;
+ private final Queue<Task<?>> runnable = new ConcurrentLinkedQueue<Task<?>>();;
+ private final List<TaskRunner> running = new ArrayList<TaskRunner>();
+ private final Map<String, StatsTask> statsTasks = new HashMap<>(1);
+ private final Context ctx;
// how many jobs have been started
private int curJobNo;
-
- private Context ctx;
private boolean shutdown;
- final Map<String, StatsTask> statsTasks = new HashMap<>(1);
-
- public DriverContext() {
+ public TaskQueue() {
+ this(null);
}
- public DriverContext(Context ctx) {
- this.runnable = new ConcurrentLinkedQueue<Task<?>>();
- this.running = new LinkedBlockingQueue<TaskRunner>();
+ public TaskQueue(Context ctx) {
this.ctx = ctx;
}
@@ -129,8 +123,9 @@ public class DriverContext {
throw new HiveException("FAILED: Operation cancelled");
}
}
+
/**
- * Cleans up remaining tasks in case of failure
+ * Cleans up remaining tasks in case of failure.
*/
public synchronized void shutdown() {
LOG.debug("Shutting down query " + ctx.getCmd());
@@ -142,7 +137,7 @@ public class DriverContext {
try {
task.shutdown();
} catch (Exception e) {
- console.printError("Exception on shutting down task " + task.getId() + ": " + e);
+ CONSOLE.printError("Exception on shutting down task " + task.getId() + ": " + e);
}
Thread thread = runner.getRunner();
if (thread != null) {
@@ -181,10 +176,6 @@ public class DriverContext {
return curJobNo;
}
- public Context getCtx() {
- return ctx;
- }
-
public void incCurJobNo(int amount) {
this.curJobNo = this.curJobNo + amount;
}
@@ -202,9 +193,6 @@ public class DriverContext {
});
}
- public void prepare(TaskRunner runner) {
- }
-
public void finished(TaskRunner runner) {
if (statsTasks.isEmpty() || !(runner.getTask() instanceof MapRedTask)) {
return;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/DDLOperationContext.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/DDLOperationContext.java
index aedbfb0..e98f5d3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/DDLOperationContext.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/DDLOperationContext.java
@@ -19,7 +19,7 @@
package org.apache.hadoop.hive.ql.ddl;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.DriverContext;
+import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.QueryPlan;
import org.apache.hadoop.hive.ql.QueryState;
import org.apache.hadoop.hive.ql.metadata.Hive;
@@ -34,7 +34,7 @@ import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
public class DDLOperationContext {
private final Hive db;
private final HiveConf conf;
- private final DriverContext driverContext;
+ private final Context context;
private final MetaDataFormatter formatter;
private final DDLTask task;
private final DDLWork work;
@@ -42,11 +42,11 @@ public class DDLOperationContext {
private final QueryPlan queryPlan;
private final LogHelper console;
- public DDLOperationContext(HiveConf conf, DriverContext driverContext, DDLTask task, DDLWork work,
- QueryState queryState, QueryPlan queryPlan, LogHelper console) throws HiveException {
+ public DDLOperationContext(HiveConf conf, Context context, DDLTask task, DDLWork work, QueryState queryState,
+ QueryPlan queryPlan, LogHelper console) throws HiveException {
this.db = Hive.get(conf);
this.conf = conf;
- this.driverContext = driverContext;
+ this.context = context;
this.formatter = MetaDataFormatUtils.getFormatter(conf);
this.task = task;
this.work = work;
@@ -63,8 +63,8 @@ public class DDLOperationContext {
return conf;
}
- public DriverContext getDriverContext() {
- return driverContext;
+ public Context getContext() {
+ return context;
}
public MetaDataFormatter getFormatter() {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/DDLTask.java
index 831d86c..3e44964 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/DDLTask.java
@@ -26,10 +26,6 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Set;
-import org.apache.hadoop.hive.ql.CompilationOpContext;
-import org.apache.hadoop.hive.ql.DriverContext;
-import org.apache.hadoop.hive.ql.QueryPlan;
-import org.apache.hadoop.hive.ql.QueryState;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.parse.ExplainConfiguration.AnalyzeState;
import org.apache.hadoop.hive.ql.plan.api.StageType;
@@ -66,14 +62,8 @@ public final class DDLTask extends Task<DDLWork> implements Serializable {
}
@Override
- public void initialize(QueryState queryState, QueryPlan queryPlan, DriverContext ctx,
- CompilationOpContext opContext) {
- super.initialize(queryState, queryPlan, ctx, opContext);
- }
-
- @Override
- public int execute(DriverContext driverContext) {
- if (driverContext.getCtx().getExplainAnalyze() == AnalyzeState.RUNNING) {
+ public int execute() {
+ if (context.getExplainAnalyze() == AnalyzeState.RUNNING) {
return 0;
}
@@ -81,12 +71,12 @@ public final class DDLTask extends Task<DDLWork> implements Serializable {
DDLDesc ddlDesc = work.getDDLDesc();
if (DESC_TO_OPARATION.containsKey(ddlDesc.getClass())) {
- DDLOperationContext context = new DDLOperationContext(conf, driverContext, this, (DDLWork)work, queryState,
- queryPlan, console);
+ DDLOperationContext ddlOperationContext = new DDLOperationContext(conf, context, this, (DDLWork)work,
+ queryState, queryPlan, console);
Class<? extends DDLOperation> ddlOpertaionClass = DESC_TO_OPARATION.get(ddlDesc.getClass());
Constructor<? extends DDLOperation> constructor =
ddlOpertaionClass.getConstructor(DDLOperationContext.class, ddlDesc.getClass());
- DDLOperation ddlOperation = constructor.newInstance(context, ddlDesc);
+ DDLOperation ddlOperation = constructor.newInstance(ddlOperationContext, ddlDesc);
return ddlOperation.execute();
} else {
throw new IllegalArgumentException("Unknown DDL request: " + ddlDesc.getClass());
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/database/lock/LockDatabaseOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/database/lock/LockDatabaseOperation.java
index 776e15e..b6c35d4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/database/lock/LockDatabaseOperation.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/database/lock/LockDatabaseOperation.java
@@ -34,7 +34,7 @@ public class LockDatabaseOperation extends DDLOperation<LockDatabaseDesc> {
@Override
public int execute() throws HiveException {
- Context ctx = context.getDriverContext().getCtx();
+ Context ctx = context.getContext();
HiveTxnManager txnManager = ctx.getHiveTxnManager();
return txnManager.lockDatabase(context.getDb(), desc);
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/database/unlock/UnlockDatabaseOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/database/unlock/UnlockDatabaseOperation.java
index 398fb84..a4f458d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/database/unlock/UnlockDatabaseOperation.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/database/unlock/UnlockDatabaseOperation.java
@@ -34,7 +34,7 @@ public class UnlockDatabaseOperation extends DDLOperation<UnlockDatabaseDesc> {
@Override
public int execute() throws HiveException {
- Context ctx = context.getDriverContext().getCtx();
+ Context ctx = context.getContext();
HiveTxnManager txnManager = ctx.getHiveTxnManager();
return txnManager.unlockDatabase(context.getDb(), desc);
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/lock/LockTableOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/lock/LockTableOperation.java
index db307e6..101d02b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/lock/LockTableOperation.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/lock/LockTableOperation.java
@@ -34,7 +34,7 @@ public class LockTableOperation extends DDLOperation<LockTableDesc> {
@Override
public int execute() throws HiveException {
- Context ctx = context.getDriverContext().getCtx();
+ Context ctx = context.getContext();
HiveTxnManager txnManager = ctx.getHiveTxnManager();
return txnManager.lockTable(context.getDb(), desc);
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/lock/ShowLocksOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/lock/ShowLocksOperation.java
index 69414f2..506b7cf 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/lock/ShowLocksOperation.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/lock/ShowLocksOperation.java
@@ -57,7 +57,7 @@ public class ShowLocksOperation extends DDLOperation<ShowLocksDesc> {
@Override
public int execute() throws HiveException {
- Context ctx = context.getDriverContext().getCtx();
+ Context ctx = context.getContext();
HiveTxnManager txnManager = ctx.getHiveTxnManager();
HiveLockManager lockMgr = txnManager.getLockManager();
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/lock/UnlockTableOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/lock/UnlockTableOperation.java
index 3759b72..a5129c8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/lock/UnlockTableOperation.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/lock/UnlockTableOperation.java
@@ -34,7 +34,7 @@ public class UnlockTableOperation extends DDLOperation<UnlockTableDesc> {
@Override
public int execute() throws HiveException {
- Context ctx = context.getDriverContext().getCtx();
+ Context ctx = context.getContext();
HiveTxnManager txnManager = ctx.getHiveTxnManager();
return txnManager.unlockTable(context.getDb(), desc);
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/TruncateTableOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/TruncateTableOperation.java
index 1244380..3c62b0a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/TruncateTableOperation.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/TruncateTableOperation.java
@@ -24,7 +24,7 @@ import java.util.Map;
import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.ql.ddl.DDLOperationContext;
import org.apache.hadoop.hive.ql.ddl.DDLUtils;
-import org.apache.hadoop.hive.ql.DriverContext;
+import org.apache.hadoop.hive.ql.TaskQueue;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.ddl.DDLOperation;
import org.apache.hadoop.hive.ql.io.rcfile.truncate.ColumnTruncateTask;
@@ -48,13 +48,13 @@ public class TruncateTableOperation extends DDLOperation<TruncateTableDesc> {
truncateWork.setListBucketingCtx(desc.getLbCtx());
truncateWork.setMapperCannotSpanPartns(true);
- DriverContext driverCxt = new DriverContext();
+ TaskQueue taskQueue = new TaskQueue();
ColumnTruncateTask taskExec = new ColumnTruncateTask();
- taskExec.initialize(context.getQueryState(), null, driverCxt, null);
+ taskExec.initialize(context.getQueryState(), null, taskQueue, null);
taskExec.setWork(truncateWork);
taskExec.setQueryPlan(context.getQueryPlan());
- int ret = taskExec.execute(driverCxt);
+ int ret = taskExec.execute();
if (taskExec.getException() != null) {
context.getTask().setException(taskExec.getException());
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableArchiveOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableArchiveOperation.java
index 315857b..5c5dec4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableArchiveOperation.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableArchiveOperation.java
@@ -175,7 +175,7 @@ public class AlterTableArchiveOperation extends DDLOperation<AlterTableArchiveDe
private Path createArchiveInTmpDir(Table table, PartSpecInfo partitionSpecInfo, Path originalDir)
throws HiveException {
// First create the archive in a tmp dir so that if the job fails, the bad files don't pollute the filesystem
- Path tmpPath = new Path(context.getDriverContext().getCtx().getExternalTmpPath(originalDir), "partlevel");
+ Path tmpPath = new Path(context.getContext().getExternalTmpPath(originalDir), "partlevel");
// Create the Hadoop archive
context.getConsole().printInfo("Creating " + ARCHIVE_NAME + " for " + originalDir.toString() + " in " + tmpPath);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableConcatenateOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableConcatenateOperation.java
index 718c21d..ea21e02 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableConcatenateOperation.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableConcatenateOperation.java
@@ -25,7 +25,8 @@ import java.util.Map;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.ql.CompilationOpContext;
-import org.apache.hadoop.hive.ql.DriverContext;
+import org.apache.hadoop.hive.ql.Context;
+import org.apache.hadoop.hive.ql.TaskQueue;
import org.apache.hadoop.hive.ql.ddl.DDLOperation;
import org.apache.hadoop.hive.ql.ddl.DDLOperationContext;
import org.apache.hadoop.hive.ql.exec.Operator;
@@ -55,11 +56,11 @@ public class AlterTableConcatenateOperation extends DDLOperation<AlterTableConca
@Override
public int execute() throws HiveException {
- CompilationOpContext opContext = context.getDriverContext().getCtx().getOpContext();
+ Context generalContext = context.getContext();
- MergeFileWork mergeWork = getMergeFileWork(opContext);
+ MergeFileWork mergeWork = getMergeFileWork(generalContext.getOpContext());
Task<?> task = getTask(mergeWork);
- return executeTask(opContext, task);
+ return executeTask(generalContext, task);
}
private MergeFileWork getMergeFileWork(CompilationOpContext opContext) {
@@ -122,10 +123,10 @@ public class AlterTableConcatenateOperation extends DDLOperation<AlterTableConca
}
}
- private int executeTask(CompilationOpContext opContext, Task<?> task) {
- DriverContext driverCxt = new DriverContext();
- task.initialize(context.getQueryState(), context.getQueryPlan(), driverCxt, opContext);
- int ret = task.execute(driverCxt);
+ private int executeTask(Context generalContext, Task<?> task) {
+ TaskQueue taskQueue = new TaskQueue();
+ task.initialize(context.getQueryState(), context.getQueryPlan(), taskQueue, generalContext);
+ int ret = task.execute();
if (task.getException() != null) {
context.getTask().setException(task.getException());
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableUnarchiveOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableUnarchiveOperation.java
index 0fbaea0..4f791a3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableUnarchiveOperation.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableUnarchiveOperation.java
@@ -97,7 +97,7 @@ public class AlterTableUnarchiveOperation extends DDLOperation<AlterTableUnarchi
throw new HiveException("Haven't found any archive where it should be");
}
- Path tmpPath = context.getDriverContext().getCtx().getExternalTmpPath(originalDir);
+ Path tmpPath = context.getContext().getExternalTmpPath(originalDir);
FileSystem fs = null;
try {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/update/MaterializedViewUpdateOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/update/MaterializedViewUpdateOperation.java
index 636b9d3..5c9ad4c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/update/MaterializedViewUpdateOperation.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/update/MaterializedViewUpdateOperation.java
@@ -40,7 +40,7 @@ public class MaterializedViewUpdateOperation extends DDLOperation<MaterializedVi
@Override
public int execute() throws HiveException {
- if (context.getDriverContext().getCtx().getExplainAnalyze() == AnalyzeState.RUNNING) {
+ if (context.getContext().getExplainAnalyze() == AnalyzeState.RUNNING) {
return 0;
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsUpdateTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsUpdateTask.java
index 526b57a..dc6d31a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsUpdateTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsUpdateTask.java
@@ -47,10 +47,6 @@ import org.apache.hadoop.hive.metastore.columnstats.cache.DoubleColumnStatsDataI
import org.apache.hadoop.hive.metastore.columnstats.cache.LongColumnStatsDataInspector;
import org.apache.hadoop.hive.metastore.columnstats.cache.StringColumnStatsDataInspector;
import org.apache.hadoop.hive.metastore.columnstats.cache.TimestampColumnStatsDataInspector;
-import org.apache.hadoop.hive.ql.CompilationOpContext;
-import org.apache.hadoop.hive.ql.DriverContext;
-import org.apache.hadoop.hive.ql.QueryPlan;
-import org.apache.hadoop.hive.ql.QueryState;
import org.apache.hadoop.hive.ql.exec.repl.util.ReplUtils;
import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.hive.ql.metadata.Hive;
@@ -77,12 +73,6 @@ public class ColumnStatsUpdateTask extends Task<ColumnStatsUpdateWork> {
private static transient final Logger LOG = LoggerFactory
.getLogger(ColumnStatsUpdateTask.class);
- @Override
- public void initialize(QueryState queryState, QueryPlan queryPlan, DriverContext ctx,
- CompilationOpContext opContext) {
- super.initialize(queryState, queryPlan, ctx, opContext);
- }
-
private ColumnStatistics constructColumnStatsFromInput()
throws SemanticException, MetaException {
@@ -366,7 +356,7 @@ public class ColumnStatsUpdateTask extends Task<ColumnStatsUpdateWork> {
}
@Override
- public int execute(DriverContext driverContext) {
+ public int execute() {
try {
Hive db = getHive();
return persistColumnStats(db);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ConditionalTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ConditionalTask.java
index ecdf368..90e552f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ConditionalTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ConditionalTask.java
@@ -21,7 +21,6 @@ package org.apache.hadoop.hive.ql.exec;
import java.io.Serializable;
import java.util.List;
-import org.apache.hadoop.hive.ql.DriverContext;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.ConditionalResolver;
import org.apache.hadoop.hive.ql.plan.ConditionalWork;
@@ -75,12 +74,12 @@ public class ConditionalTask extends Task<ConditionalWork> implements Serializab
}
@Override
- public int execute(DriverContext driverContext) {
+ public int execute() {
resTasks = resolver.getTasks(conf, resolverCtx);
resolved = true;
try {
- resolveTask(driverContext);
+ resolveTask();
} catch (Exception e) {
setException(e);
return 1;
@@ -88,13 +87,13 @@ public class ConditionalTask extends Task<ConditionalWork> implements Serializab
return 0;
}
- private void resolveTask(DriverContext driverContext) throws HiveException {
+ private void resolveTask() throws HiveException {
for (Task<?> tsk : getListTasks()) {
if (!resTasks.contains(tsk)) {
- driverContext.remove(tsk);
+ taskQueue.remove(tsk);
console.printInfo(tsk.getId() + " is filtered out by condition resolver.");
if (tsk.isMapRedTask()) {
- driverContext.incCurJobNo(1);
+ taskQueue.incCurJobNo(1);
}
//recursively remove this task from its children's parent task
tsk.removeFromChildrenTasks();
@@ -106,7 +105,7 @@ public class ConditionalTask extends Task<ConditionalWork> implements Serializab
}
}
// resolved task
- if (driverContext.addToRunnable(tsk)) {
+ if (taskQueue.addToRunnable(tsk)) {
console.printInfo(tsk.getId() + " is selected by condition resolver.");
}
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java
index 1a8e5e7..c11f582 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java
@@ -18,22 +18,13 @@
package org.apache.hadoop.hive.ql.exec;
-import java.io.IOException;
import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
-import org.apache.hadoop.hive.common.JavaUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.hive.common.FileUtils;
-import org.apache.hadoop.hive.ql.DriverContext;
-import org.apache.hadoop.hive.ql.io.AcidUtils;
-import org.apache.hadoop.hive.ql.parse.repl.dump.io.FileOperations;
import org.apache.hadoop.hive.ql.plan.CopyWork;
import org.apache.hadoop.hive.ql.plan.api.StageType;
import org.apache.hadoop.util.StringUtils;
@@ -44,14 +35,12 @@ import org.apache.hadoop.util.StringUtils;
public class CopyTask extends Task<CopyWork> implements Serializable {
private static final long serialVersionUID = 1L;
- private static transient final Logger LOG = LoggerFactory.getLogger(CopyTask.class);
-
public CopyTask() {
super();
}
@Override
- public int execute(DriverContext driverContext) {
+ public int execute() {
Path[] from = work.getFromPaths(), to = work.getToPaths();
for (int i = 0; i < from.length; ++i) {
int result = copyOnePath(from[i], to[i]);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DependencyCollectionTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DependencyCollectionTask.java
index 95b8205..b889379 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DependencyCollectionTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DependencyCollectionTask.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hive.ql.exec;
import java.io.Serializable;
-import org.apache.hadoop.hive.ql.DriverContext;
import org.apache.hadoop.hive.ql.plan.DependencyCollectionWork;
import org.apache.hadoop.hive.ql.plan.api.StageType;
@@ -39,7 +38,7 @@ public class DependencyCollectionTask extends Task<DependencyCollectionWork>
}
@Override
- public int execute(DriverContext driverContext) {
+ public int execute() {
return 0;
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainSQRewriteTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainSQRewriteTask.java
index 1f9e9aa..1929fab 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainSQRewriteTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainSQRewriteTask.java
@@ -29,7 +29,6 @@ import java.util.Set;
import org.antlr.runtime.TokenRewriteStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.ql.DriverContext;
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.HiveParser;
import org.apache.hadoop.hive.ql.parse.QB;
@@ -41,7 +40,6 @@ import org.apache.hadoop.io.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
public class ExplainSQRewriteTask extends Task<ExplainSQRewriteWork> implements Serializable {
private static final long serialVersionUID = 1L;
private final Logger LOG = LoggerFactory.getLogger(this.getClass().getName());
@@ -52,7 +50,7 @@ public class ExplainSQRewriteTask extends Task<ExplainSQRewriteWork> implements
}
@Override
- public int execute(DriverContext driverContext) {
+ public int execute() {
PrintStream out = null;
try {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
index a0f3e15..4d79ebc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
@@ -46,7 +46,6 @@ import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.LockComponent;
-import org.apache.hadoop.hive.ql.DriverContext;
import org.apache.hadoop.hive.ql.hooks.ReadEntity;
import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.hive.ql.metadata.Table;
@@ -398,7 +397,7 @@ public class ExplainTask extends Task<ExplainWork> implements Serializable {
}
@Override
- public int execute(DriverContext driverContext) {
+ public int execute() {
PrintStream out = null;
try {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExportTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExportTask.java
index ae9d040..ebc590d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExportTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExportTask.java
@@ -17,7 +17,6 @@
*/
package org.apache.hadoop.hive.ql.exec;
-import org.apache.hadoop.hive.ql.DriverContext;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.parse.SemanticException;
@@ -44,7 +43,7 @@ public class ExportTask extends Task<ExportWork> implements Serializable {
}
@Override
- public int execute(DriverContext driverContext) {
+ public int execute() {
try {
// Also creates the root directory
TableExport.Paths exportPaths = new TableExport.Paths(
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java
index 93b1158..502d41d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java
@@ -24,8 +24,8 @@ import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.hadoop.hive.ql.CompilationOpContext;
-import org.apache.hadoop.hive.ql.DriverContext;
+import org.apache.hadoop.hive.ql.Context;
+import org.apache.hadoop.hive.ql.TaskQueue;
import org.apache.hadoop.hive.ql.QueryPlan;
import org.apache.hadoop.hive.ql.QueryState;
import org.apache.hadoop.hive.ql.exec.mr.ExecMapper;
@@ -62,10 +62,9 @@ public class FetchTask extends Task<FetchWork> implements Serializable {
}
@Override
- public void initialize(QueryState queryState, QueryPlan queryPlan, DriverContext ctx,
- CompilationOpContext opContext) {
- super.initialize(queryState, queryPlan, ctx, opContext);
- work.initializeForFetch(opContext);
+ public void initialize(QueryState queryState, QueryPlan queryPlan, TaskQueue taskQueue, Context context) {
+ super.initialize(queryState, queryPlan, taskQueue, context);
+ work.initializeForFetch(context.getOpContext());
try {
// Create a file system handle
@@ -109,7 +108,7 @@ public class FetchTask extends Task<FetchWork> implements Serializable {
}
@Override
- public int execute(DriverContext driverContext) {
+ public int execute() {
assert false;
return 0;
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
index 695d08b..06e4ebe 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
@@ -29,8 +29,6 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
import org.apache.hadoop.hive.metastore.api.Order;
import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
-import org.apache.hadoop.hive.ql.Context;
-import org.apache.hadoop.hive.ql.DriverContext;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.ddl.DDLUtils;
import org.apache.hadoop.hive.ql.exec.mr.MapRedTask;
@@ -229,15 +227,14 @@ public class MoveTask extends Task<MoveWork> implements Serializable {
return;
}
- Context ctx = driverContext.getCtx();
- if (ctx.getHiveTxnManager().supportsAcid()) {
+ if (context.getHiveTxnManager().supportsAcid()) {
//Acid LM doesn't maintain getOutputLockObjects(); this 'if' just makes logic more explicit
return;
}
- HiveLockManager lockMgr = ctx.getHiveTxnManager().getLockManager();
- WriteEntity output = ctx.getLoadTableOutputMap().get(ltd);
- List<HiveLockObj> lockObjects = ctx.getOutputLockObjects().get(output);
+ HiveLockManager lockMgr = context.getHiveTxnManager().getLockManager();
+ WriteEntity output = context.getLoadTableOutputMap().get(ltd);
+ List<HiveLockObj> lockObjects = context.getOutputLockObjects().get(output);
if (CollectionUtils.isEmpty(lockObjects)) {
LOG.debug("No locks found to release");
return;
@@ -248,7 +245,7 @@ public class MoveTask extends Task<MoveWork> implements Serializable {
List<HiveLock> locks = lockMgr.getLocks(lockObj.getObj(), false, true);
for (HiveLock lock : locks) {
if (lock.getHiveLockMode() == lockObj.getMode()) {
- if (ctx.getHiveLocks().remove(lock)) {
+ if (context.getHiveLocks().remove(lock)) {
try {
lockMgr.unlock(lock);
} catch (LockException le) {
@@ -302,7 +299,7 @@ public class MoveTask extends Task<MoveWork> implements Serializable {
}
@Override
- public int execute(DriverContext driverContext) {
+ public int execute() {
if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
Utilities.FILE_OP_LOGGER.trace("Executing MoveWork " + System.identityHashCode(work)
+ " with " + work.getLoadFileWork() + "; " + work.getLoadTableWork() + "; "
@@ -310,7 +307,7 @@ public class MoveTask extends Task<MoveWork> implements Serializable {
}
try {
- if (driverContext.getCtx().getExplainAnalyze() == AnalyzeState.RUNNING) {
+ if (context.getExplainAnalyze() == AnalyzeState.RUNNING) {
return 0;
}
Hive db = getHive();
@@ -404,7 +401,7 @@ public class MoveTask extends Task<MoveWork> implements Serializable {
throw new HiveException("MoveTask : Write id is not set in the config by open txn task for migration");
}
tbd.setWriteId(writeId);
- tbd.setStmtId(driverContext.getCtx().getHiveTxnManager().getStmtIdAndIncrement());
+ tbd.setStmtId(context.getHiveTxnManager().getStmtIdAndIncrement());
}
boolean isFullAcidOp = work.getLoadTableWork().getWriteType() != AcidUtils.Operation.NOT_ACID
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java
index 6b3635d..470357a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java
@@ -45,7 +45,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.DriverContext;
import org.apache.hadoop.hive.ql.parse.LoadSemanticAnalyzer;
import org.apache.hadoop.hive.ql.plan.api.StageType;
import org.apache.hadoop.util.StringUtils;
@@ -133,7 +132,7 @@ public class ReplCopyTask extends Task<ReplCopyWork> implements Serializable {
}
@Override
- public int execute(DriverContext driverContext) {
+ public int execute() {
LOG.debug("ReplCopyTask.execute()");
FileSystem dstFs = null;
Path toPath = null;
@@ -215,7 +214,7 @@ public class ReplCopyTask extends Task<ReplCopyWork> implements Serializable {
// duplicate copy from the source. Check HIVE-21197 for more detail.
int stmtId = (writeId.equals(ReplUtils.REPL_BOOTSTRAP_MIGRATION_BASE_WRITE_ID)) ?
ReplUtils.REPL_BOOTSTRAP_MIGRATION_BASE_STMT_ID :
- driverContext.getCtx().getHiveTxnManager().getStmtIdAndIncrement();
+ context.getHiveTxnManager().getStmtIdAndIncrement();
toPath = new Path(toPath, AcidUtils.baseOrDeltaSubdir(work.getDeleteDestIfExist(), writeId, writeId, stmtId));
}
} else {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplTxnTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplTxnTask.java
index c91b78e..9131aee 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplTxnTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplTxnTask.java
@@ -18,11 +18,9 @@
package org.apache.hadoop.hive.ql.exec;
-import org.apache.hadoop.hive.common.ValidWriteIdList;
import org.apache.hadoop.hive.metastore.api.CommitTxnRequest;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.TxnToWriteId;
-import org.apache.hadoop.hive.ql.DriverContext;
import org.apache.hadoop.hive.ql.exec.repl.util.ReplUtils;
import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager;
import org.apache.hadoop.hive.ql.metadata.Hive;
@@ -50,7 +48,7 @@ public class ReplTxnTask extends Task<ReplTxnWork> {
}
@Override
- public int execute(DriverContext driverContext) {
+ public int execute() {
String replPolicy = work.getReplPolicy();
String tableName = work.getTableName();
ReplicationSpec replicationSpec = work.getReplicationSpec();
@@ -85,30 +83,30 @@ public class ReplTxnTask extends Task<ReplTxnWork> {
}
try {
- HiveTxnManager txnManager = driverContext.getCtx().getHiveTxnManager();
+ HiveTxnManager txnManager = context.getHiveTxnManager();
String user = UserGroupInformation.getCurrentUser().getUserName();
switch(work.getOperationType()) {
case REPL_OPEN_TXN:
List<Long> txnIds = txnManager.replOpenTxn(replPolicy, work.getTxnIds(), user);
assert txnIds.size() == work.getTxnIds().size();
LOG.info("Replayed OpenTxn Event for policy " + replPolicy + " with srcTxn " +
- work.getTxnIds().toString() + " and target txn id " + txnIds.toString());
+ work.getTxnIds().toString() + " and target txn id " + txnIds.toString());
return 0;
case REPL_MIGRATION_OPEN_TXN:
- // if transaction is already opened (mostly by repl load command), then close it.
- if (txnManager.isTxnOpen()) {
- long txnId = txnManager.getCurrentTxnId();
- txnManager.commitTxn();
- LOG.info("Committed txn from REPL_MIGRATION_OPEN_TXN : " + txnId);
- }
- Long txnIdMigration = txnManager.openTxn(driverContext.getCtx(), user);
- long writeId = txnManager.getTableWriteId(work.getDbName(), work.getTableName());
- String validTxnList = txnManager.getValidTxns().toString();
- conf.set(ValidTxnList.VALID_TXNS_KEY, validTxnList);
- conf.set(ReplUtils.REPL_CURRENT_TBL_WRITE_ID, Long.toString(writeId));
- LOG.info("Started open txn for migration : " + txnIdMigration + " with valid txn list : " +
- validTxnList + " and write id " + writeId);
- return 0;
+ // if transaction is already opened (mostly by repl load command), then close it.
+ if (txnManager.isTxnOpen()) {
+ long txnId = txnManager.getCurrentTxnId();
+ txnManager.commitTxn();
+ LOG.info("Committed txn from REPL_MIGRATION_OPEN_TXN : " + txnId);
+ }
+ Long txnIdMigration = txnManager.openTxn(context, user);
+ long writeId = txnManager.getTableWriteId(work.getDbName(), work.getTableName());
+ String validTxnList = txnManager.getValidTxns().toString();
+ conf.set(ValidTxnList.VALID_TXNS_KEY, validTxnList);
+ conf.set(ReplUtils.REPL_CURRENT_TBL_WRITE_ID, Long.toString(writeId));
+ LOG.info("Started open txn for migration : " + txnIdMigration + " with valid txn list : " +
+ validTxnList + " and write id " + writeId);
+ return 0;
case REPL_ABORT_TXN:
for (long txnId : work.getTxnIds()) {
txnManager.replRollbackTxn(replPolicy, txnId);
@@ -116,16 +114,16 @@ public class ReplTxnTask extends Task<ReplTxnWork> {
}
return 0;
case REPL_MIGRATION_COMMIT_TXN:
- assert (work.getReplLastIdInfo() != null);
- long txnIdMigrationCommit = txnManager.getCurrentTxnId();
- CommitTxnRequest commitTxnRequestMigr = new CommitTxnRequest(txnIdMigrationCommit);
- commitTxnRequestMigr.setReplLastIdInfo(work.getReplLastIdInfo());
- txnManager.replCommitTxn(commitTxnRequestMigr);
- conf.unset(ValidTxnList.VALID_TXNS_KEY);
- conf.unset(ReplUtils.REPL_CURRENT_TBL_WRITE_ID);
- LOG.info("Committed Migration Txn with replLastIdInfo: " + work.getReplLastIdInfo() + " for txnId: " +
- txnIdMigrationCommit);
- return 0;
+ assert (work.getReplLastIdInfo() != null);
+ long txnIdMigrationCommit = txnManager.getCurrentTxnId();
+ CommitTxnRequest commitTxnRequestMigr = new CommitTxnRequest(txnIdMigrationCommit);
+ commitTxnRequestMigr.setReplLastIdInfo(work.getReplLastIdInfo());
+ txnManager.replCommitTxn(commitTxnRequestMigr);
+ conf.unset(ValidTxnList.VALID_TXNS_KEY);
+ conf.unset(ReplUtils.REPL_CURRENT_TBL_WRITE_ID);
+ LOG.info("Committed Migration Txn with replLastIdInfo: " + work.getReplLastIdInfo() + " for txnId: " +
+ txnIdMigrationCommit);
+ return 0;
case REPL_COMMIT_TXN:
// Currently only one commit txn per event is supported.
assert (work.getTxnIds().size() == 1);
@@ -136,7 +134,7 @@ public class ReplTxnTask extends Task<ReplTxnWork> {
commitTxnRequest.setWriteEventInfos(work.getWriteEventInfos());
txnManager.replCommitTxn(commitTxnRequest);
LOG.info("Replayed CommitTxn Event for replPolicy: " + replPolicy + " with srcTxn: " + txnId +
- "WriteEventInfos: " + work.getWriteEventInfos());
+ "WriteEventInfos: " + work.getWriteEventInfos());
return 0;
case REPL_ALLOC_WRITE_ID:
assert work.getTxnToWriteIdList() != null;
@@ -144,14 +142,13 @@ public class ReplTxnTask extends Task<ReplTxnWork> {
List <TxnToWriteId> txnToWriteIdList = work.getTxnToWriteIdList();
txnManager.replAllocateTableWriteIdsBatch(dbName, tableName, replPolicy, txnToWriteIdList);
LOG.info("Replayed alloc write Id Event for repl policy: " + replPolicy + " db Name : " + dbName +
- " txnToWriteIdList: " +txnToWriteIdList.toString() + " table name: " + tableName);
+ " txnToWriteIdList: " +txnToWriteIdList.toString() + " table name: " + tableName);
return 0;
case REPL_WRITEID_STATE:
- txnManager.replTableWriteIdState(work.getValidWriteIdList(),
- work.getDbName(), tableName, work.getPartNames());
- LOG.info("Replicated WriteId state for DbName: " + work.getDbName()
- + " TableName: " + tableName
- + " ValidWriteIdList: " + work.getValidWriteIdList());
+ txnManager.replTableWriteIdState(work.getValidWriteIdList(), work.getDbName(), tableName, work.getPartNames());
+ LOG.info("Replicated WriteId state for DbName: " + work.getDbName() +
+ " TableName: " + tableName +
+ " ValidWriteIdList: " + work.getValidWriteIdList());
return 0;
default:
LOG.error("Operation Type " + work.getOperationType() + " is not supported ");
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java
index 7a4242a..393d21d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java
@@ -27,10 +27,10 @@ import java.util.concurrent.Executors;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.ql.CompilationOpContext;
+import org.apache.hadoop.hive.ql.Context;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.hadoop.hive.ql.DriverContext;
+import org.apache.hadoop.hive.ql.TaskQueue;
import org.apache.hadoop.hive.ql.QueryPlan;
import org.apache.hadoop.hive.ql.QueryState;
import org.apache.hadoop.hive.ql.metadata.Hive;
@@ -63,9 +63,8 @@ public class StatsTask extends Task<StatsWork> implements Serializable {
List<IStatsProcessor> processors = new ArrayList<>();
@Override
- public void initialize(QueryState queryState, QueryPlan queryPlan, DriverContext ctx,
- CompilationOpContext opContext) {
- super.initialize(queryState, queryPlan, ctx, opContext);
+ public void initialize(QueryState queryState, QueryPlan queryPlan, TaskQueue taskQueue, Context context) {
+ super.initialize(queryState, queryPlan, taskQueue, context);
if (work.getBasicStatsWork() != null) {
BasicStatsTask task = new BasicStatsTask(conf, work.getBasicStatsWork());
@@ -80,14 +79,14 @@ public class StatsTask extends Task<StatsWork> implements Serializable {
}
for (IStatsProcessor p : processors) {
- p.initialize(opContext);
+ p.initialize(context.getOpContext());
}
}
@Override
- public int execute(DriverContext driverContext) {
- if (driverContext.getCtx().getExplainAnalyze() == AnalyzeState.RUNNING) {
+ public int execute() {
+ if (context.getExplainAnalyze() == AnalyzeState.RUNNING) {
return 0;
}
if (work.isAggregating() && work.isFooterScan()) {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java
index 8eea9cf..86a2964 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java
@@ -20,8 +20,8 @@ package org.apache.hadoop.hive.ql.exec;
import org.apache.hadoop.hive.common.metrics.common.Metrics;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.CompilationOpContext;
-import org.apache.hadoop.hive.ql.DriverContext;
+import org.apache.hadoop.hive.ql.Context;
+import org.apache.hadoop.hive.ql.TaskQueue;
import org.apache.hadoop.hive.ql.QueryDisplay;
import org.apache.hadoop.hive.ql.QueryPlan;
import org.apache.hadoop.hive.ql.QueryState;
@@ -61,7 +61,8 @@ public abstract class Task<T extends Serializable> implements Serializable, Node
protected transient QueryState queryState;
protected transient LogHelper console;
protected transient QueryPlan queryPlan;
- protected transient DriverContext driverContext;
+ protected transient TaskQueue taskQueue;
+ protected transient Context context;
protected transient boolean clonedConf = false;
protected transient String jobID;
protected Task<?> backupTask;
@@ -142,7 +143,7 @@ public abstract class Task<T extends Serializable> implements Serializable, Node
protected List<Task<?>> parentTasks;
/**
* this can be set by the Task, to provide more info about the failure in TaskResult
- * where the Driver can find it. This is checked if {@link Task#execute(org.apache.hadoop.hive.ql.DriverContext)}
+ * where the Driver can find it. This is checked if {@link Task#execute(org.apache.hadoop.hive.ql.TaskQueue)}
* returns non-0 code.
*/
private Throwable exception;
@@ -156,16 +157,16 @@ public abstract class Task<T extends Serializable> implements Serializable, Node
return taskHandle;
}
- public void initialize(QueryState queryState, QueryPlan queryPlan, DriverContext driverContext,
- CompilationOpContext opContext) {
+ public void initialize(QueryState queryState, QueryPlan queryPlan, TaskQueue taskQueue, Context context) {
this.queryPlan = queryPlan;
setInitialized();
this.queryState = queryState;
- if (null == this.conf) {
+ if (null == this.conf && queryState != null) {
this.conf = queryState.getConf();
}
- this.driverContext = driverContext;
- console = new LogHelper(LOG);
+ this.taskQueue = taskQueue;
+ this.context = context;
+ this.console = new LogHelper(LOG);
}
public void setQueryDisplay(QueryDisplay queryDisplay) {
this.queryDisplay = queryDisplay;
@@ -209,7 +210,7 @@ public abstract class Task<T extends Serializable> implements Serializable, Node
if (conf != null) {
LOG.debug("Task getting executed using mapred tag : " + conf.get(MRJobConfig.JOB_TAGS));
}
- int retval = execute(driverContext);
+ int retval = execute();
this.setDone();
if (hiveHistory != null) {
hiveHistory.logPlanProgress(queryPlan);
@@ -225,7 +226,7 @@ public abstract class Task<T extends Serializable> implements Serializable, Node
*
* @return status of executing the task
*/
- public abstract int execute(DriverContext driverContext);
+ public abstract int execute();
public boolean isRootTask() {
return rootTask;
@@ -594,12 +595,16 @@ public abstract class Task<T extends Serializable> implements Serializable, Node
return queryPlan;
}
- public DriverContext getDriverContext() {
- return driverContext;
+ public TaskQueue getTaskQueue() {
+ return taskQueue;
}
- public void setDriverContext(DriverContext driverContext) {
- this.driverContext = driverContext;
+ public void setTaskQueue(TaskQueue taskQueue) {
+ this.taskQueue = taskQueue;
+ }
+
+ public Context getContext() {
+ return context;
}
public void setQueryPlan(QueryPlan queryPlan) {
@@ -657,6 +662,6 @@ public abstract class Task<T extends Serializable> implements Serializable, Node
}
public HiveTxnManager getTxnMgr() {
- return driverContext.getCtx().getHiveTxnManager();
+ return context.getHiveTxnManager();
}
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskRunner.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskRunner.java
index 6f1ab97..16b9710 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskRunner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskRunner.java
@@ -22,7 +22,7 @@ import java.io.Serializable;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.hadoop.hive.common.LogUtils;
-import org.apache.hadoop.hive.ql.DriverContext;
+import org.apache.hadoop.hive.ql.TaskQueue;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.slf4j.Logger;
@@ -48,13 +48,13 @@ public class TaskRunner extends Thread {
private static transient final Logger LOG = LoggerFactory.getLogger(TaskRunner.class);
- private final DriverContext driverCtx;
+ private final TaskQueue taskQueue;
- public TaskRunner(Task<?> tsk, DriverContext ctx) {
+ public TaskRunner(Task<?> tsk, TaskQueue taskQueue) {
this.tsk = tsk;
this.result = new TaskResult();
- ss = SessionState.get();
- driverCtx = ctx;
+ this.ss = SessionState.get();
+ this.taskQueue = taskQueue;
}
public Task<?> getTask() {
@@ -110,7 +110,7 @@ public class TaskRunner extends Thread {
LOG.error("Error in executeTask", t);
}
result.setExitVal(exitVal);
- driverCtx.releaseRunnable();
+ taskQueue.releaseRunnable();
if (tsk.getException() != null) {
result.setTaskError(tsk.getException());
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
index 9a2ca76..196ca39 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
@@ -52,9 +52,8 @@ import org.apache.hadoop.hive.common.LogUtils.LogInitializationException;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.conf.HiveConfUtil;
-import org.apache.hadoop.hive.ql.CompilationOpContext;
import org.apache.hadoop.hive.ql.Context;
-import org.apache.hadoop.hive.ql.DriverContext;
+import org.apache.hadoop.hive.ql.TaskQueue;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.QueryPlan;
import org.apache.hadoop.hive.ql.QueryState;
@@ -173,9 +172,8 @@ public class ExecDriver extends Task<MapredWork> implements Serializable, Hadoop
* Initialization when invoked from QL.
*/
@Override
- public void initialize(QueryState queryState, QueryPlan queryPlan, DriverContext driverContext,
- CompilationOpContext opContext) {
- super.initialize(queryState, queryPlan, driverContext, opContext);
+ public void initialize(QueryState queryState, QueryPlan queryPlan, TaskQueue taskQueue, Context context) {
+ super.initialize(queryState, queryPlan, taskQueue, context);
job = new JobConf(conf, ExecDriver.class);
@@ -220,19 +218,18 @@ public class ExecDriver extends Task<MapredWork> implements Serializable, Hadoop
*/
@SuppressWarnings({"deprecation", "unchecked"})
@Override
- public int execute(DriverContext driverContext) {
+ public int execute() {
IOPrepareCache ioPrepareCache = IOPrepareCache.get();
ioPrepareCache.clear();
boolean success = true;
- Context ctx = driverContext.getCtx();
boolean ctxCreated = false;
Path emptyScratchDir;
JobClient jc = null;
- if (driverContext.isShutdown()) {
+ if (taskQueue.isShutdown()) {
LOG.warn("Task was cancelled");
return 5;
}
@@ -240,6 +237,7 @@ public class ExecDriver extends Task<MapredWork> implements Serializable, Hadoop
MapWork mWork = work.getMapWork();
ReduceWork rWork = work.getReduceWork();
+ Context ctx = context;
try {
if (ctx == null) {
ctx = new Context(job);
@@ -412,14 +410,14 @@ public class ExecDriver extends Task<MapredWork> implements Serializable, Hadoop
HiveConfUtil.updateJobCredentialProviders(job);
// Finally SUBMIT the JOB!
- if (driverContext.isShutdown()) {
+ if (taskQueue.isShutdown()) {
LOG.warn("Task was cancelled");
return 5;
}
rj = jc.submitJob(job);
- if (driverContext.isShutdown()) {
+ if (taskQueue.isShutdown()) {
LOG.warn("Task was cancelled");
killJob();
return 5;
@@ -768,12 +766,14 @@ public class ExecDriver extends Task<MapredWork> implements Serializable, Hadoop
memoryMXBean = ManagementFactory.getMemoryMXBean();
MapredLocalWork plan = SerializationUtilities.deserializePlan(pathData, MapredLocalWork.class);
MapredLocalTask ed = new MapredLocalTask(plan, conf, isSilent);
- ret = ed.executeInProcess(new DriverContext());
+ ed.initialize(null, null, new TaskQueue(), null);
+ ret = ed.executeInProcess();
} else {
MapredWork plan = SerializationUtilities.deserializePlan(pathData, MapredWork.class);
ExecDriver ed = new ExecDriver(plan, conf, isSilent);
- ret = ed.execute(new DriverContext());
+ ed.setTaskQueue(new TaskQueue());
+ ret = ed.execute();
}
if (ret != 0) {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java
index 0f594a1..dffc5cb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java
@@ -37,7 +37,6 @@ import org.apache.hadoop.hive.common.metrics.common.Metrics;
import org.apache.hadoop.hive.common.metrics.common.MetricsConstant;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.Context;
-import org.apache.hadoop.hive.ql.DriverContext;
import org.apache.hadoop.hive.ql.MapRedStats;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.SerializationUtilities;
@@ -92,10 +91,10 @@ public class MapRedTask extends ExecDriver implements Serializable {
}
@Override
- public int execute(DriverContext driverContext) {
+ public int execute() {
- Context ctx = driverContext.getCtx();
boolean ctxCreated = false;
+ Context ctx = context;
try {
if (ctx == null) {
@@ -111,7 +110,7 @@ public class MapRedTask extends ExecDriver implements Serializable {
conf.getBoolVar(HiveConf.ConfVars.LOCALMODEAUTO)) {
if (inputSummary == null) {
- inputSummary = Utilities.getInputSummary(driverContext.getCtx(), work.getMapWork(), null);
+ inputSummary = Utilities.getInputSummary(ctx, work.getMapWork(), null);
}
// set the values of totalInputFileSize and totalInputNumFiles, estimating them
@@ -156,7 +155,7 @@ public class MapRedTask extends ExecDriver implements Serializable {
}
// we are not running this mapred task via child jvm
// so directly invoke ExecDriver
- int ret = super.execute(driverContext);
+ int ret = super.execute();
// restore the previous properties for framework name, RM address etc.
if (this.isLocalMode()) {
@@ -324,7 +323,7 @@ public class MapRedTask extends ExecDriver implements Serializable {
try {
// creating the context can create a bunch of files. So make
// sure to clear it out
- if(ctxCreated) {
+ if (ctxCreated) {
ctx.clear();
}
@@ -442,7 +441,7 @@ public class MapRedTask extends ExecDriver implements Serializable {
+ reducers);
} else {
if (inputSummary == null) {
- inputSummary = Utilities.getInputSummary(driverContext.getCtx(), work.getMapWork(), null);
+ inputSummary = Utilities.getInputSummary(context, work.getMapWork(), null);
}
int reducers = Utilities.estimateNumberOfReducers(conf, inputSummary, work.getMapWork(),
work.isFinalMapRed());
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java
index 4bc7568..9d499c0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java
@@ -48,9 +48,8 @@ import org.apache.hadoop.hive.common.metrics.common.Metrics;
import org.apache.hadoop.hive.common.metrics.common.MetricsConstant;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.ql.CompilationOpContext;
import org.apache.hadoop.hive.ql.Context;
-import org.apache.hadoop.hive.ql.DriverContext;
+import org.apache.hadoop.hive.ql.TaskQueue;
import org.apache.hadoop.hive.ql.QueryPlan;
import org.apache.hadoop.hive.ql.QueryState;
import org.apache.hadoop.hive.ql.exec.BucketMatcher;
@@ -132,9 +131,8 @@ public class MapredLocalTask extends Task<MapredLocalWork> implements Serializab
}
@Override
- public void initialize(QueryState queryState, QueryPlan queryPlan, DriverContext driverContext,
- CompilationOpContext opContext) {
- super.initialize(queryState, queryPlan, driverContext, opContext);
+ public void initialize(QueryState queryState, QueryPlan queryPlan, TaskQueue taskQueue, Context context) {
+ super.initialize(queryState, queryPlan, taskQueue, context);
job = new JobConf(conf, ExecDriver.class);
execContext = new ExecMapperContext(job);
//we don't use the HadoopJobExecHooks for local tasks
@@ -153,27 +151,26 @@ public class MapredLocalTask extends Task<MapredLocalWork> implements Serializab
}
@Override
- public int execute(DriverContext driverContext) {
+ public int execute() {
if (conf.getBoolVar(HiveConf.ConfVars.SUBMITLOCALTASKVIACHILD)) {
// send task off to another jvm
- return executeInChildVM(driverContext);
+ return executeInChildVM();
} else {
// execute in process
- return executeInProcess(driverContext);
+ return executeInProcess();
}
}
- public int executeInChildVM(DriverContext driverContext) {
+ private int executeInChildVM() {
// execute in child jvm
try {
// generate the cmd line to run in the child jvm
- Context ctx = driverContext.getCtx();
String hiveJar = conf.getJar();
String hadoopExec = conf.getVar(HiveConf.ConfVars.HADOOPBIN);
conf.setVar(ConfVars.HIVEADDEDJARS, Utilities.getResourceFiles(conf, SessionState.ResourceType.JAR));
// write out the plan to a local file
- Path planPath = new Path(ctx.getLocalTmpPath(), "plan.xml");
+ Path planPath = new Path(context.getLocalTmpPath(), "plan.xml");
MapredLocalWork plan = getWork();
LOG.info("Generating plan file " + planPath.toString());
@@ -193,7 +190,7 @@ public class MapredLocalTask extends Task<MapredLocalWork> implements Serializab
String libJarsOption = StringUtils.isEmpty(libJars) ? " " : " -libjars " + libJars + " ";
String jarCmd = hiveJar + " " + ExecDriver.class.getName() + libJarsOption;
- String hiveConfArgs = ExecDriver.generateCmdLine(conf, ctx);
+ String hiveConfArgs = ExecDriver.generateCmdLine(conf, context);
String cmdLine = hadoopExec + " jar " + jarCmd + " -localtask -plan " + planPath.toString()
+ " " + isSilent + " " + hiveConfArgs;
@@ -203,7 +200,7 @@ public class MapredLocalTask extends Task<MapredLocalWork> implements Serializab
if (!files.isEmpty()) {
cmdLine = cmdLine + " -files " + files;
- workDir = ctx.getLocalTmpPath().toUri().getPath();
+ workDir = context.getLocalTmpPath().toUri().getPath();
if (!(new File(workDir)).mkdir()) {
throw new IOException("Cannot create tmp working dir: " + workDir);
@@ -370,7 +367,7 @@ public class MapredLocalTask extends Task<MapredLocalWork> implements Serializab
}
}
- public int executeInProcess(DriverContext driverContext) {
+ public int executeInProcess() {
// check the local work
if (work == null) {
return -1;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ExternalTableCopyTaskBuilder.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ExternalTableCopyTaskBuilder.java
index 1af9227..5c5543c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ExternalTableCopyTaskBuilder.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ExternalTableCopyTaskBuilder.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hive.ql.exec.repl;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.DriverContext;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.TaskFactory;
@@ -163,7 +162,7 @@ public class ExternalTableCopyTaskBuilder {
}
@Override
- public int execute(DriverContext driverContext) {
+ public int execute() {
String distCpDoAsUser = conf.getVar(HiveConf.ConfVars.HIVE_DISTCP_DOAS_USER);
Path sourcePath = work.fullyQualifiedSourcePath;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java
index 32cb38a..622433b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java
@@ -38,7 +38,6 @@ import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint;
import org.apache.hadoop.hive.metastore.messaging.event.filters.AndFilter;
import org.apache.hadoop.hive.metastore.messaging.event.filters.EventBoundaryFilter;
import org.apache.hadoop.hive.metastore.messaging.event.filters.ReplEventFilter;
-import org.apache.hadoop.hive.ql.DriverContext;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.repl.util.ReplUtils;
@@ -117,7 +116,7 @@ public class ReplDumpTask extends Task<ReplDumpWork> implements Serializable {
}
@Override
- public int execute(DriverContext driverContext) {
+ public int execute() {
try {
Hive hiveDb = getHive();
Path dumpRoot = new Path(conf.getVar(HiveConf.ConfVars.REPLDIR), getNextDumpDir());
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java
index 9a541d2..7e5a40a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.hive.common.repl.ReplScope;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.InvalidInputException;
-import org.apache.hadoop.hive.ql.DriverContext;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.ddl.DDLWork;
import org.apache.hadoop.hive.ql.ddl.database.alter.poperties.AlterDatabaseSetPropertiesDesc;
@@ -97,7 +96,7 @@ public class ReplLoadTask extends Task<ReplLoadWork> implements Serializable {
}
@Override
- public int execute(DriverContext driverContext) {
+ public int execute() {
Task<?> rootTask = work.getRootTask();
if (rootTask != null) {
rootTask.setChildTasks(null);
@@ -105,17 +104,17 @@ public class ReplLoadTask extends Task<ReplLoadWork> implements Serializable {
work.setRootTask(this);
this.parentTasks = null;
if (work.isIncrementalLoad()) {
- return executeIncrementalLoad(driverContext);
+ return executeIncrementalLoad();
} else {
- return executeBootStrapLoad(driverContext);
+ return executeBootStrapLoad();
}
}
- private int executeBootStrapLoad(DriverContext driverContext) {
+ private int executeBootStrapLoad() {
try {
int maxTasks = conf.getIntVar(HiveConf.ConfVars.REPL_APPROX_MAX_LOAD_TASKS);
- Context context = new Context(work.dumpDirectory, conf, getHive(),
- work.sessionStateLineageState, driverContext.getCtx());
+ Context loadContext = new Context(work.dumpDirectory, conf, getHive(),
+ work.sessionStateLineageState, context);
TaskTracker loadTaskTracker = new TaskTracker(maxTasks);
/*
for now for simplicity we are doing just one directory ( one database ), come back to use
@@ -155,10 +154,10 @@ public class ReplLoadTask extends Task<ReplLoadWork> implements Serializable {
switch (next.eventType()) {
case Database:
DatabaseEvent dbEvent = (DatabaseEvent) next;
- dbTracker = new LoadDatabase(context, dbEvent, work.dbNameToLoadIn, loadTaskTracker).tasks();
+ dbTracker = new LoadDatabase(loadContext, dbEvent, work.dbNameToLoadIn, loadTaskTracker).tasks();
loadTaskTracker.update(dbTracker);
if (work.hasDbState()) {
- loadTaskTracker.update(updateDatabaseLastReplID(maxTasks, context, scope));
+ loadTaskTracker.update(updateDatabaseLastReplID(maxTasks, loadContext, scope));
} else {
// Scope might have set to database in some previous iteration of loop, so reset it to false if database
// tracker has no tasks.
@@ -180,7 +179,7 @@ public class ReplLoadTask extends Task<ReplLoadWork> implements Serializable {
*/
TableContext tableContext = new TableContext(dbTracker, work.dbNameToLoadIn);
TableEvent tableEvent = (TableEvent) next;
- LoadTable loadTable = new LoadTable(tableEvent, context, iterator.replLogger(),
+ LoadTable loadTable = new LoadTable(tableEvent, loadContext, iterator.replLogger(),
tableContext, loadTaskTracker);
tableTracker = loadTable.tasks(work.isIncrementalLoad());
setUpDependencies(dbTracker, tableTracker);
@@ -203,7 +202,7 @@ public class ReplLoadTask extends Task<ReplLoadWork> implements Serializable {
// for a table we explicitly try to load partitions as there is no separate partitions events.
LoadPartitions loadPartitions =
- new LoadPartitions(context, iterator.replLogger(), loadTaskTracker, tableEvent,
+ new LoadPartitions(loadContext, iterator.replLogger(), loadTaskTracker, tableEvent,
work.dbNameToLoadIn, tableContext);
TaskTracker partitionsTracker = loadPartitions.tasks();
partitionsPostProcessing(iterator, scope, loadTaskTracker, tableTracker,
@@ -220,7 +219,7 @@ public class ReplLoadTask extends Task<ReplLoadWork> implements Serializable {
PartitionEvent event = (PartitionEvent) next;
TableContext tableContext = new TableContext(dbTracker, work.dbNameToLoadIn);
LoadPartitions loadPartitions =
- new LoadPartitions(context, iterator.replLogger(), tableContext, loadTaskTracker,
+ new LoadPartitions(loadContext, iterator.replLogger(), tableContext, loadTaskTracker,
event.asTableEvent(), work.dbNameToLoadIn, event.lastPartitionReplicated());
/*
the tableTracker here should be a new instance and not an existing one as this can
@@ -233,7 +232,7 @@ public class ReplLoadTask extends Task<ReplLoadWork> implements Serializable {
break;
}
case Function: {
- LoadFunction loadFunction = new LoadFunction(context, iterator.replLogger(),
+ LoadFunction loadFunction = new LoadFunction(loadContext, iterator.replLogger(),
(FunctionEvent) next, work.dbNameToLoadIn, dbTracker);
TaskTracker functionsTracker = loadFunction.tasks();
if (!scope.database) {
@@ -247,7 +246,7 @@ public class ReplLoadTask extends Task<ReplLoadWork> implements Serializable {
}
case Constraint: {
LoadConstraint loadConstraint =
- new LoadConstraint(context, (ConstraintEvent) next, work.dbNameToLoadIn, dbTracker);
+ new LoadConstraint(loadContext, (ConstraintEvent) next, work.dbNameToLoadIn, dbTracker);
TaskTracker constraintTracker = loadConstraint.tasks();
scope.rootTasks.addAll(constraintTracker.tasks());
loadTaskTracker.update(constraintTracker);
@@ -256,7 +255,7 @@ public class ReplLoadTask extends Task<ReplLoadWork> implements Serializable {
}
if (!loadingConstraint && !iterator.currentDbHasNext()) {
- createEndReplLogTask(context, scope, iterator.replLogger());
+ createEndReplLogTask(loadContext, scope, iterator.replLogger());
}
}
@@ -274,7 +273,7 @@ public class ReplLoadTask extends Task<ReplLoadWork> implements Serializable {
// last repl ID of the database.
if (!iterator.hasNext() && !constraintIterator.hasNext() && !work.getPathsToCopyIterator().hasNext()
&& !work.isIncrementalLoad()) {
- loadTaskTracker.update(updateDatabaseLastReplID(maxTasks, context, scope));
+ loadTaskTracker.update(updateDatabaseLastReplID(maxTasks, loadContext, scope));
work.updateDbEventState(null);
}
this.childTasks = scope.rootTasks;
@@ -286,7 +285,7 @@ public class ReplLoadTask extends Task<ReplLoadWork> implements Serializable {
LOG.info("Root Tasks / Total Tasks : {} / {} ", childTasks.size(), loadTaskTracker.numberOfTasks());
// Populate the driver context with the scratch dir info from the repl context, so that the temp dirs will be cleaned up later
- driverContext.getCtx().getFsScratchDirs().putAll(context.pathInfo.getFsScratchDirs());
+ context.getFsScratchDirs().putAll(loadContext.pathInfo.getFsScratchDirs());
} catch (RuntimeException e) {
LOG.error("replication failed with run time exception", e);
throw e;
@@ -484,7 +483,7 @@ public class ReplLoadTask extends Task<ReplLoadWork> implements Serializable {
DAGTraversal.traverse(rootTasks, new AddDependencyToLeaves(loadTask));
}
- private int executeIncrementalLoad(DriverContext driverContext) {
+ private int executeIncrementalLoad() {
try {
// If user has requested to cleanup any bootstrap dump, then just do it before incremental load.
if (work.needCleanTablesFromBootstrap) {
@@ -503,7 +502,7 @@ public class ReplLoadTask extends Task<ReplLoadWork> implements Serializable {
if (work.hasBootstrapLoadTasks()) {
LOG.debug("Current incremental dump have tables to be bootstrapped. Switching to bootstrap "
+ "mode after applying all events.");
- return executeBootStrapLoad(driverContext);
+ return executeBootStrapLoad();
}
}
@@ -517,7 +516,7 @@ public class ReplLoadTask extends Task<ReplLoadWork> implements Serializable {
if (work.getPathsToCopyIterator().hasNext()) {
childTasks.addAll(new ExternalTableCopyTaskBuilder(work, conf).tasks(tracker));
} else {
- childTasks.add(builder.build(driverContext, getHive(), LOG, tracker));
+ childTasks.add(builder.build(context, getHive(), LOG, tracker));
}
// If there are no more events to be applied, add a task to update the last.repl.id of the
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplStateLogTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplStateLogTask.java
index 845aad1..7ade7c0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplStateLogTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplStateLogTask.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hive.ql.exec.repl;
-import org.apache.hadoop.hive.ql.DriverContext;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.plan.api.StageType;
@@ -34,7 +33,7 @@ public class ReplStateLogTask extends Task<ReplStateLogWork> implements Serializ
private static final long serialVersionUID = 1L;
@Override
- public int execute(DriverContext driverContext) {
+ public int execute() {
work.replStateLog();
return 0;
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/incremental/IncrementalLoadTasksBuilder.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/incremental/IncrementalLoadTasksBuilder.java
index 09e4f31..d7b6eea 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/incremental/IncrementalLoadTasksBuilder.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/incremental/IncrementalLoadTasksBuilder.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.ReplLastIdInfo;
-import org.apache.hadoop.hive.ql.DriverContext;
+import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.ddl.DDLWork;
import org.apache.hadoop.hive.ql.ddl.database.alter.poperties.AlterDatabaseSetPropertiesDesc;
import org.apache.hadoop.hive.ql.ddl.misc.flags.ReplRemoveFirstIncLoadPendFlagDesc;
@@ -53,7 +53,6 @@ import org.apache.hadoop.hive.ql.plan.DependencyCollectionWork;
import org.apache.hadoop.hive.ql.plan.ReplTxnWork;
import org.slf4j.Logger;
-import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -89,7 +88,7 @@ public class IncrementalLoadTasksBuilder {
numIteration = 0;
}
- public Task<?> build(DriverContext driverContext, Hive hive, Logger log,
+ public Task<?> build(Context context, Hive hive, Logger log,
TaskTracker tracker) throws Exception {
Task<?> evTaskRoot = TaskFactory.get(new DependencyCollectionWork());
Task<?> taskChainTail = evTaskRoot;
@@ -131,9 +130,9 @@ public class IncrementalLoadTasksBuilder {
// Once this entire chain is generated, we add evTaskRoot to rootTasks, so as to execute the
// entire chain
- MessageHandler.Context context = new MessageHandler.Context(dbName, location,
- taskChainTail, eventDmd, conf, hive, driverContext.getCtx(), this.log);
- List<Task<?>> evTasks = analyzeEventLoad(context);
+ MessageHandler.Context mhContext = new MessageHandler.Context(dbName, location,
+ taskChainTail, eventDmd, conf, hive, context, this.log);
+ List<Task<?>> evTasks = analyzeEventLoad(mhContext);
if ((evTasks != null) && (!evTasks.isEmpty())) {
ReplStateLogWork replStateLogWork = new ReplStateLogWork(replLogger,
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/schq/ScheduledQueryMaintenanceTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/schq/ScheduledQueryMaintenanceTask.java
index 318e238..3d46b18 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/schq/ScheduledQueryMaintenanceTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/schq/ScheduledQueryMaintenanceTask.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.hive.ql.exec.schq;
import org.apache.hadoop.hive.metastore.api.ScheduledQueryMaintenanceRequest;
-import org.apache.hadoop.hive.ql.DriverContext;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -42,7 +41,7 @@ public class ScheduledQueryMaintenanceTask extends Task<ScheduledQueryMaintenanc
}
@Override
- public int execute(DriverContext driverContext) {
+ public int execute() {
ScheduledQueryMaintenanceRequest request = buildScheduledQueryRequest();
try {
Hive.get().getMSC().scheduledQueryMaintenance(request);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClient.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClient.java
index 8a44c1b..454e407 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClient.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClient.java
@@ -20,21 +20,22 @@ package org.apache.hadoop.hive.ql.exec.spark;
import java.io.Closeable;
import java.io.Serializable;
-import org.apache.hadoop.hive.ql.DriverContext;
+import org.apache.hadoop.hive.ql.Context;
+import org.apache.hadoop.hive.ql.TaskQueue;
import org.apache.hadoop.hive.ql.exec.spark.status.SparkJobRef;
import org.apache.hadoop.hive.ql.plan.SparkWork;
import org.apache.spark.SparkConf;
public interface HiveSparkClient extends Serializable, Closeable {
/**
- * HiveSparkClient should generate Spark RDD graph by given sparkWork and driverContext,
+ * HiveSparkClient should generate Spark RDD graph by given sparkWork and taskQueue,
* and submit RDD graph to Spark cluster.
- * @param driverContext
+ * @param taskQueue
* @param sparkWork
* @return SparkJobRef could be used to track spark job progress and metrics.
* @throws Exception
*/
- SparkJobRef execute(DriverContext driverContext, SparkWork sparkWork) throws Exception;
+ SparkJobRef execute(TaskQueue taskQueue, Context context, SparkWork sparkWork) throws Exception;
/**
* @return spark configuration
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java
index 6c6122a..2b6bce0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java
@@ -34,7 +34,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConfUtil;
import org.apache.hadoop.hive.ql.Context;
-import org.apache.hadoop.hive.ql.DriverContext;
+import org.apache.hadoop.hive.ql.TaskQueue;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.exec.spark.status.SparkJobRef;
import org.apache.hadoop.hive.ql.exec.spark.status.impl.JobMetricsListener;
@@ -124,15 +124,14 @@ public class LocalHiveSparkClient implements HiveSparkClient {
}
@Override
- public SparkJobRef execute(DriverContext driverContext, SparkWork sparkWork) throws Exception {
- Context ctx = driverContext.getCtx();
- HiveConf hiveConf = (HiveConf) ctx.getConf();
+ public SparkJobRef execute(TaskQueue taskQueue, Context context, SparkWork sparkWork) throws Exception {
+ HiveConf hiveConf = (HiveConf) context.getConf();
refreshLocalResources(sparkWork, hiveConf);
JobConf jobConf = new JobConf(hiveConf);
// Create temporary scratch dir
Path emptyScratchDir;
- emptyScratchDir = ctx.getMRTmpPath();
+ emptyScratchDir = context.getMRTmpPath();
FileSystem fs = emptyScratchDir.getFileSystem(jobConf);
fs.mkdirs(emptyScratchDir);
@@ -153,11 +152,10 @@ public class LocalHiveSparkClient implements HiveSparkClient {
SparkReporter sparkReporter = new SparkReporter(sparkCounters);
// Generate Spark plan
- SparkPlanGenerator gen =
- new SparkPlanGenerator(sc, ctx, jobConf, emptyScratchDir, sparkReporter);
+ SparkPlanGenerator gen = new SparkPlanGenerator(sc, context, jobConf, emptyScratchDir, sparkReporter);
SparkPlan plan = gen.generate(sparkWork);
- if (driverContext.isShutdown()) {
+ if (taskQueue.isShutdown()) {
throw new HiveException("Operation is cancelled.");
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
index 49b7614..f23db78 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
@@ -23,7 +23,6 @@ import com.google.common.base.Strings;
import java.io.IOException;
import java.io.Serializable;
-import java.io.StringWriter;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
@@ -37,7 +36,6 @@ import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.ql.io.NullScanFileSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.FileUtils;
@@ -45,7 +43,7 @@ import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.conf.HiveConfUtil;
import org.apache.hadoop.hive.ql.Context;
-import org.apache.hadoop.hive.ql.DriverContext;
+import org.apache.hadoop.hive.ql.TaskQueue;
import org.apache.hadoop.hive.ql.exec.DagUtils;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.exec.spark.status.SparkJobRef;
@@ -181,8 +179,7 @@ public class RemoteHiveSparkClient implements HiveSparkClient {
}
@Override
- public SparkJobRef execute(final DriverContext driverContext, final SparkWork sparkWork)
- throws Exception {
+ public SparkJobRef execute(TaskQueue taskQueue, Context context, SparkWork sparkWork) throws Exception {
if (SparkClientUtilities.isYarnMaster(hiveConf.get("spark.master")) &&
!remoteClient.isActive()) {
// Re-create the remote client if not active any more
@@ -191,15 +188,14 @@ public class RemoteHiveSparkClient implements HiveSparkClient {
}
try {
- return submit(driverContext, sparkWork);
+ return submit(taskQueue, context, sparkWork);
} catch (Throwable cause) {
throw new Exception("Failed to submit Spark work, please retry later", cause);
}
}
- private SparkJobRef submit(final DriverContext driverContext, final SparkWork sparkWork) throws Exception {
- final Context ctx = driverContext.getCtx();
- final HiveConf hiveConf = (HiveConf) ctx.getConf();
+ private SparkJobRef submit(TaskQueue taskQueue, Context context, SparkWork sparkWork) throws Exception {
+ final HiveConf hiveConf = (HiveConf) context.getConf();
refreshLocalResources(sparkWork, hiveConf);
final JobConf jobConf = new JobConf(hiveConf);
@@ -207,7 +203,7 @@ public class RemoteHiveSparkClient implements HiveSparkClient {
HiveConfUtil.updateJobCredentialProviders(jobConf);
// Create temporary scratch dir
- final Path emptyScratchDir = ctx.getMRTmpPath();
+ final Path emptyScratchDir = context.getMRTmpPath();
FileSystem fs = emptyScratchDir.getFileSystem(jobConf);
fs.mkdirs(emptyScratchDir);
@@ -220,7 +216,7 @@ public class RemoteHiveSparkClient implements HiveSparkClient {
byte[] sparkWorkBytes = KryoSerializer.serialize(sparkWork);
JobStatusJob job = new JobStatusJob(jobConfBytes, scratchDirBytes, sparkWorkBytes);
- if (driverContext.isShutdown()) {
+ if (taskQueue.isShutdown()) {
throw new HiveException("Operation is cancelled.");
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java
index 9277510..f401b4d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java
@@ -44,11 +44,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.CompilationOpContext;
-import org.apache.hadoop.hive.ql.DriverContext;
import org.apache.hadoop.hive.ql.ErrorMsg;
-import org.apache.hadoop.hive.ql.QueryPlan;
-import org.apache.hadoop.hive.ql.QueryState;
import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
import org.apache.hadoop.hive.ql.exec.JoinOperator;
import org.apache.hadoop.hive.ql.exec.MapOperator;
@@ -104,13 +100,7 @@ public class SparkTask extends Task<SparkWork> {
private transient boolean jobKilled = false;
@Override
- public void initialize(QueryState queryState, QueryPlan queryPlan, DriverContext driverContext,
- CompilationOpContext opContext) {
- super.initialize(queryState, queryPlan, driverContext, opContext);
- }
-
- @Override
- public int execute(DriverContext driverContext) {
+ public int execute() {
int rc = 0;
perfLogger = SessionState.getPerfLogger();
@@ -127,11 +117,11 @@ public class SparkTask extends Task<SparkWork> {
// Submit the Spark job
perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.SPARK_SUBMIT_JOB);
submitTime = perfLogger.getStartTime(PerfLogger.SPARK_SUBMIT_JOB);
- jobRef = sparkSession.submit(driverContext, sparkWork);
+ jobRef = sparkSession.submit(taskQueue, context, sparkWork);
perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.SPARK_SUBMIT_JOB);
// If the driver context has been shutdown (due to query cancellation) kill the Spark job
- if (driverContext.isShutdown()) {
+ if (taskQueue.isShutdown()) {
LOG.warn("Killing Spark job");
killJob();
throw new HiveException(String.format("Spark task %s cancelled for query %s", getId(), sparkWork.getQueryId()));
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSession.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSession.java
index b29be48..e19b68c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSession.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSession.java
@@ -20,7 +20,8 @@ package org.apache.hadoop.hive.ql.exec.spark.session;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.DriverContext;
+import org.apache.hadoop.hive.ql.Context;
+import org.apache.hadoop.hive.ql.TaskQueue;
import org.apache.hadoop.hive.ql.exec.spark.status.SparkJobRef;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.SparkWork;
@@ -37,11 +38,11 @@ public interface SparkSession {
/**
* Submit given <i>sparkWork</i> to SparkClient.
- * @param driverContext
+ * @param taskQueue
* @param sparkWork
* @return SparkJobRef
*/
- SparkJobRef submit(DriverContext driverContext, SparkWork sparkWork) throws Exception;
+ SparkJobRef submit(TaskQueue taskQueue, Context context, SparkWork sparkWork) throws Exception;
/**
* Get Spark shuffle memory per task, and total number of cores. This
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java
index 7e64342..d4a36b2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java
@@ -36,6 +36,7 @@ import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.session.SessionState;
@@ -43,7 +44,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.DriverContext;
+import org.apache.hadoop.hive.ql.TaskQueue;
import org.apache.hadoop.hive.ql.exec.spark.HiveSparkClient;
import org.apache.hadoop.hive.ql.exec.spark.HiveSparkClientFactory;
import org.apache.hadoop.hive.ql.exec.spark.status.SparkJobRef;
@@ -142,11 +143,11 @@ public class SparkSessionImpl implements SparkSession {
}
@Override
- public SparkJobRef submit(DriverContext driverContext, SparkWork sparkWork) throws Exception {
+ public SparkJobRef submit(TaskQueue taskQueue, Context context, SparkWork sparkWork) throws Exception {
closeLock.readLock().lock();
try {
Preconditions.checkState(isOpen, "Hive on Spark session is not open. Can't submit jobs.");
- return hiveSparkClient.execute(driverContext, sparkWork);
+ return hiveSparkClient.execute(taskQueue, context, sparkWork);
} finally {
closeLock.readLock().unlock();
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
index aecd108..0c8695f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
@@ -40,7 +40,6 @@ import org.apache.hadoop.hive.common.metrics.common.Metrics;
import org.apache.hadoop.hive.common.metrics.common.MetricsConstant;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.Context;
-import org.apache.hadoop.hive.ql.DriverContext;
import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.Task;
@@ -129,7 +128,7 @@ public class TezTask extends Task<TezWork> {
@Override
- public int execute(DriverContext driverContext) {
+ public int execute() {
int rc = 1;
boolean cleanContext = false;
Context ctx = null;
@@ -137,7 +136,7 @@ public class TezTask extends Task<TezWork> {
try {
// Get or create Context object. If we create it we have to clean it later as well.
- ctx = driverContext.getCtx();
+ ctx = context;
if (ctx == null) {
ctx = new Context(conf);
cleanContext = true;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecWMEventsSummaryPrinter.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecWMEventsSummaryPrinter.java
index 94c66c0..c5905f5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecWMEventsSummaryPrinter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecWMEventsSummaryPrinter.java
@@ -52,7 +52,7 @@ public class PostExecWMEventsSummaryPrinter implements ExecuteWithHookContext {
List<TezTask> rootTasks = Utilities.getTezTasks(plan.getRootTasks());
for (TezTask tezTask : rootTasks) {
- WmContext wmContext = tezTask.getDriverContext().getCtx().getWmContext();
+ WmContext wmContext = tezTask.getContext().getWmContext();
if (wmContext != null) {
wmContext.printJson(console);
wmContext.shortPrint(console);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/merge/MergeFileTask.java b/ql/src/java/org/apache/hadoop/hive/ql/io/merge/MergeFileTask.java
index 7eb2ef3..39f77ed 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/merge/MergeFileTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/merge/MergeFileTask.java
@@ -24,9 +24,8 @@ import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.CompilationOpContext;
import org.apache.hadoop.hive.ql.Context;
-import org.apache.hadoop.hive.ql.DriverContext;
+import org.apache.hadoop.hive.ql.TaskQueue;
import org.apache.hadoop.hive.ql.QueryPlan;
import org.apache.hadoop.hive.ql.QueryState;
import org.apache.hadoop.hive.ql.exec.Operator;
@@ -63,9 +62,8 @@ public class MergeFileTask extends Task<MergeFileWork> implements Serializable,
private boolean success = true;
@Override
- public void initialize(QueryState queryState, QueryPlan queryPlan,
- DriverContext driverContext, CompilationOpContext opContext) {
- super.initialize(queryState, queryPlan, driverContext, opContext);
+ public void initialize(QueryState queryState, QueryPlan queryPlan, TaskQueue taskQueue, Context context) {
+ super.initialize(queryState, queryPlan, taskQueue, context);
job = new JobConf(conf, MergeFileTask.class);
jobExecHelper = new HadoopJobExecHelper(job, this.console, this, this);
}
@@ -79,9 +77,9 @@ public class MergeFileTask extends Task<MergeFileWork> implements Serializable,
* start a new map-reduce job to do the merge, almost the same as ExecDriver.
*/
@Override
- public int execute(DriverContext driverContext) {
+ public int execute() {
- Context ctx = driverContext.getCtx();
+ Context ctx = context;
boolean ctxCreated = false;
RunningJob rj = null;
int returnVal = 0;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java b/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java
index 17b5b4b..ae9f022 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java
@@ -27,9 +27,8 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.CompilationOpContext;
import org.apache.hadoop.hive.ql.Context;
-import org.apache.hadoop.hive.ql.DriverContext;
+import org.apache.hadoop.hive.ql.TaskQueue;
import org.apache.hadoop.hive.ql.QueryPlan;
import org.apache.hadoop.hive.ql.QueryState;
import org.apache.hadoop.hive.ql.exec.Task;
@@ -61,9 +60,8 @@ public class ColumnTruncateTask extends Task<ColumnTruncateWork> implements Seri
protected HadoopJobExecHelper jobExecHelper;
@Override
- public void initialize(QueryState queryState, QueryPlan queryPlan,
- DriverContext driverContext, CompilationOpContext opContext) {
- super.initialize(queryState, queryPlan, driverContext, opContext);
+ public void initialize(QueryState queryState, QueryPlan queryPlan, TaskQueue taskQueue, Context context) {
+ super.initialize(queryState, queryPlan, taskQueue, context);
job = new JobConf(conf, ColumnTruncateTask.class);
jobExecHelper = new HadoopJobExecHelper(job, this.console, this, this);
}
@@ -75,11 +73,11 @@ public class ColumnTruncateTask extends Task<ColumnTruncateWork> implements Seri
boolean success = true;
- @Override
/**
* start a new map-reduce job to do the truncation, almost the same as ExecDriver.
*/
- public int execute(DriverContext driverContext) {
+ @Override
+ public int execute() {
HiveConf.setVar(job, HiveConf.ConfVars.HIVEINPUTFORMAT,
BucketizedHiveInputFormat.class.getName());
success = true;
@@ -87,7 +85,7 @@ public class ColumnTruncateTask extends Task<ColumnTruncateWork> implements Seri
job.setOutputFormat(HiveOutputFormatImpl.class);
job.setMapperClass(work.getMapperClass());
- Context ctx = driverContext.getCtx();
+ Context ctx = context;
boolean ctxCreated = false;
try {
if (ctx == null) {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java
index 8257177..169f059 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java
@@ -34,7 +34,7 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.ql.Context;
-import org.apache.hadoop.hive.ql.DriverContext;
+import org.apache.hadoop.hive.ql.TaskQueue;
import org.apache.hadoop.hive.ql.QueryState;
import org.apache.hadoop.hive.ql.ddl.DDLTask;
import org.apache.hadoop.hive.ql.ddl.DDLWork;
@@ -155,7 +155,9 @@ public class AcidExportSemanticAnalyzer extends RewriteSemanticAnalyzer {
inputs.add(dbForTmpTable); //so the plan knows we are 'reading' this db - locks, security...
DDLTask createTableTask = (DDLTask) TaskFactory.get(new DDLWork(new HashSet<>(), new HashSet<>(), ctlt), conf);
createTableTask.setConf(conf); //above get() doesn't set it
- createTableTask.execute(new DriverContext(new Context(conf)));
+ Context context = new Context(conf);
+ createTableTask.initialize(null, null, new TaskQueue(context), context);
+ createTableTask.execute();
newTable = db.getTable(newTableName);
} catch(IOException|HiveException ex) {
throw new SemanticException(ex);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java
index e9c10e4..dd49e3a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java
@@ -152,7 +152,7 @@ public class ExplainSemanticAnalyzer extends BaseSemanticAnalyzer {
while (driver.getResults(new ArrayList<String>())) {
}
} catch (CommandProcessorException e) {
- throw new SemanticException(e.getErrorMessage(), e.getException());
+ throw new SemanticException(e.getErrorMessage(), e);
}
config.setOpIdToRuntimeNumRows(aggregateStats(config.getExplainRootPath()));
} catch (IOException e1) {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/BasicStatsWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/BasicStatsWork.java
index 8fa50a8..20f7d2e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/BasicStatsWork.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/BasicStatsWork.java
@@ -57,7 +57,7 @@ public class BasicStatsWork implements Serializable {
// sourceTask for TS is not changed (currently) but that of FS might be changed
// by various optimizers (auto.convert.join, for example)
- // so this is set by DriverContext in runtime
+ // so this is set by TaskQueue in runtime
private transient Task sourceTask;
private boolean isFollowedByColStats = false;
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java
index a78fdfc..dd70524 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java
@@ -34,7 +34,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.CompilationOpContext;
-import org.apache.hadoop.hive.ql.DriverContext;
+import org.apache.hadoop.hive.ql.TaskQueue;
import org.apache.hadoop.hive.ql.QueryState;
import org.apache.hadoop.hive.ql.exec.mr.ExecDriver;
import org.apache.hadoop.hive.ql.exec.mr.MapRedTask;
@@ -481,10 +481,10 @@ public class TestExecDriver {
private void executePlan() throws Exception {
String testName = new Exception().getStackTrace()[1].getMethodName();
MapRedTask mrtask = new MapRedTask();
- DriverContext dctx = new DriverContext();
+ TaskQueue taskQueue = new TaskQueue();
mrtask.setWork(mr);
- mrtask.initialize(queryState, null, dctx, null);
- int exitVal = mrtask.execute(dctx);
+ mrtask.initialize(queryState, null, taskQueue, null);
+ int exitVal = mrtask.execute();
if (exitVal != 0) {
LOG.error(testName + " execution failed with exit status: "
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/mr/TestMapRedTask.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/mr/TestMapRedTask.java
index c9b68fa..6a06f4c 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/mr/TestMapRedTask.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/mr/TestMapRedTask.java
@@ -33,9 +33,8 @@ import org.apache.hadoop.hive.common.metrics.common.Metrics;
import org.apache.hadoop.hive.common.metrics.common.MetricsConstant;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.Context;
-import org.apache.hadoop.hive.ql.DriverContext;
+import org.apache.hadoop.hive.ql.TaskQueue;
import org.apache.hadoop.hive.ql.QueryState;
-import org.apache.hadoop.hive.ql.exec.spark.SparkTask;
import org.apache.hadoop.hive.ql.plan.MapWork;
import org.apache.hadoop.hive.ql.plan.MapredWork;
import org.apache.hadoop.hive.shims.Utils;
@@ -67,7 +66,7 @@ public class TestMapRedTask {
Context ctx = Mockito.mock(Context.class);
when(ctx.getLocalTmpPath()).thenReturn(new Path(System.getProperty("java.io.tmpdir")));
- DriverContext dctx = new DriverContext(ctx);
+ TaskQueue taskQueue = new TaskQueue(ctx);
QueryState queryState = new QueryState.Builder().build();
HiveConf conf= queryState.getConf();
@@ -79,12 +78,12 @@ public class TestMapRedTask {
MapRedTask mrTask = Mockito.spy(new MapRedTask());
mrTask.setWork(mrWork);
- mrTask.initialize(queryState, null, dctx, null);
+ mrTask.initialize(queryState, null, taskQueue, ctx);
mrTask.jobExecHelper = Mockito.mock(HadoopJobExecHelper.class);
when(mrTask.jobExecHelper.progressLocal(Mockito.any(Process.class), Mockito.anyString())).thenReturn(0);
- mrTask.execute(dctx);
+ mrTask.execute();
ArgumentCaptor<String[]> captor = ArgumentCaptor.forClass(String[].class);
verify(mrTask).spawn(Mockito.anyString(), Mockito.anyString(), captor.capture());
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/TestSparkTask.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/TestSparkTask.java
index 2017fc1..bde6f6d 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/TestSparkTask.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/TestSparkTask.java
@@ -32,7 +32,7 @@ import java.util.concurrent.ExecutionException;
import org.apache.hadoop.hive.common.metrics.common.Metrics;
import org.apache.hadoop.hive.common.metrics.common.MetricsConstant;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.DriverContext;
+import org.apache.hadoop.hive.ql.TaskQueue;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.QueryState;
import org.apache.hadoop.hive.ql.exec.spark.Statistic.SparkStatisticsBuilder;
@@ -195,12 +195,12 @@ public class TestSparkTask {
SparkTask sparkTask = new SparkTask();
sparkTask.setWork(mock(SparkWork.class));
- DriverContext mockDriverContext = mock(DriverContext.class);
+ TaskQueue mockTestQueue = mock(TaskQueue.class);
QueryState mockQueryState = mock(QueryState.class);
when(mockQueryState.getConf()).thenReturn(hiveConf);
- sparkTask.initialize(mockQueryState, null, mockDriverContext, null);
+ sparkTask.initialize(mockQueryState, null, mockTestQueue, null);
SparkJobStatus mockSparkJobStatus = mock(SparkJobStatus.class);
when(mockSparkJobStatus.getMonitorError()).thenReturn(new InterruptedException());
@@ -210,19 +210,19 @@ public class TestSparkTask {
when(mockSparkJobRef.monitorJob()).thenReturn(2);
when(mockSparkJobRef.getSparkJobStatus()).thenReturn(mockSparkJobStatus);
- when(mockSparkSession.submit(any(), any())).thenReturn(mockSparkJobRef);
+ when(mockSparkSession.submit(any(), any(), any())).thenReturn(mockSparkJobRef);
SessionState.start(hiveConf);
SessionState.get().setSparkSession(mockSparkSession);
- sparkTask.execute(mockDriverContext);
+ sparkTask.execute();
verify(mockSparkJobRef, atLeastOnce()).cancelJob();
when(mockSparkJobStatus.getMonitorError()).thenReturn(
new HiveException(new InterruptedException()));
- sparkTask.execute(mockDriverContext);
+ sparkTask.execute();
verify(mockSparkJobRef, atLeastOnce()).cancelJob();
}
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestMacroSemanticAnalyzer.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestMacroSemanticAnalyzer.java
index 0030649..8b5ae71 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestMacroSemanticAnalyzer.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestMacroSemanticAnalyzer.java
@@ -22,7 +22,7 @@ import java.util.List;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.ql.Context;
-import org.apache.hadoop.hive.ql.DriverContext;
+import org.apache.hadoop.hive.ql.TaskQueue;
import org.apache.hadoop.hive.ql.QueryState;
import org.apache.hadoop.hive.ql.ddl.DDLSemanticAnalyzerFactory;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
@@ -58,7 +58,7 @@ public class TestMacroSemanticAnalyzer {
List<Task<?>> rootTasks = analyzer.getRootTasks();
Assert.assertEquals(1, rootTasks.size());
for (Task<?> task : rootTasks) {
- task.setDriverContext(new DriverContext(context));
+ task.initialize(null, null, new TaskQueue(context), context);
task.setConf(conf);
Assert.assertEquals(0, task.executeTask(null));
}
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java
index e07f785..461cfdc 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java
@@ -303,7 +303,7 @@ public class TestUpdateDeleteSemanticAnalyzer {
ExplainTask task = new ExplainTask();
task.setWork(work);
task.initialize(queryState, plan, null, null);
- task.execute(null);
+ task.execute();
FSDataInputStream in = fs.open(tmp);
StringBuilder builder = new StringBuilder();
final int bufSz = 4096;