You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by hy...@apache.org on 2014/04/10 09:32:44 UTC
git commit: TAJO-563: INSERT OVERWRITE should not remove data before
query success. (hyunsik)
Repository: tajo
Updated Branches:
refs/heads/master 743c52650 -> 20cd09201
TAJO-563: INSERT OVERWRITE should not remove data before query success. (hyunsik)
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/20cd0920
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/20cd0920
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/20cd0920
Branch: refs/heads/master
Commit: 20cd09201af20c9084ba41805e0c70036471106f
Parents: 743c526
Author: Hyunsik Choi <hy...@apache.org>
Authored: Thu Apr 10 16:32:23 2014 +0900
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Thu Apr 10 16:32:23 2014 +0900
----------------------------------------------------------------------
CHANGES.txt | 3 ++
.../java/org/apache/tajo/TajoConstants.java | 1 +
.../apache/tajo/master/querymaster/Query.java | 30 ++++++++++++++++++--
.../master/querymaster/QueryMasterTask.java | 12 +-------
4 files changed, 33 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/20cd0920/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 9353512..c2d123e 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -315,6 +315,9 @@ Release 0.8.0 - unreleased
BUG FIXES
+ TAJO-563: INSERT OVERWRITE should not remove data before query success.
+ (hyunsik)
+
TAJO-738: NPE occur when failed in QueryMaster's GlobalPlanner.build().
(hyoungjunkim via hyunsik)
http://git-wip-us.apache.org/repos/asf/tajo/blob/20cd0920/tajo-common/src/main/java/org/apache/tajo/TajoConstants.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/TajoConstants.java b/tajo-common/src/main/java/org/apache/tajo/TajoConstants.java
index 61eecd1..028ca81 100644
--- a/tajo-common/src/main/java/org/apache/tajo/TajoConstants.java
+++ b/tajo-common/src/main/java/org/apache/tajo/TajoConstants.java
@@ -26,6 +26,7 @@ public class TajoConstants {
public static final String WAREHOUSE_DIR_NAME = "warehouse";
public static final String SYSTEM_RESOURCE_DIR_NAME = "resource";
public static final String RESULT_DIR_NAME="RESULT";
+ public static final String INSERT_OVERWIRTE_OLD_TABLE_NAME ="OLD_TABLE";
public static final String DEFAULT_TABLESPACE_NAME = "default";
public static final String DEFAULT_DATABASE_NAME = "default";
http://git-wip-us.apache.org/repos/asf/tajo/blob/20cd0920/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java
index 3a4df7b..a8f5b31 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java
@@ -396,8 +396,34 @@ public class Query implements EventHandler<QueryEvent> {
finalOutputDir = queryContext.getOutputPath();
try {
FileSystem fs = stagingResultDir.getFileSystem(query.systemConf);
- fs.rename(stagingResultDir, finalOutputDir);
- LOG.info("Moved from the staging dir to the output directory '" + finalOutputDir);
+
+ if (queryContext.isOutputOverwrite()) { // INSERT OVERWRITE INTO
+
+ // it moves the original table into the temporary location.
+ // Then it moves the new result table into the original table location.
+ // Upon failed, it recovers the original table if possible.
+ boolean movedToOldTable = false;
+ boolean committed = false;
+ Path oldTableDir = new Path(queryContext.getStagingDir(), TajoConstants.INSERT_OVERWIRTE_OLD_TABLE_NAME);
+ try {
+ if (fs.exists(finalOutputDir)) {
+ fs.rename(finalOutputDir, oldTableDir);
+ movedToOldTable = fs.exists(oldTableDir);
+ } else { // if the parent does not exist, make its parent directory.
+ fs.mkdirs(finalOutputDir.getParent());
+ }
+ fs.rename(stagingResultDir, finalOutputDir);
+ committed = fs.exists(finalOutputDir);
+ } catch (IOException ioe) {
+ // recover the old table
+ if (movedToOldTable && !committed) {
+ fs.rename(oldTableDir, finalOutputDir);
+ }
+ }
+ } else {
+ fs.rename(stagingResultDir, finalOutputDir);
+ LOG.info("Moved from the staging dir to the output directory '" + finalOutputDir);
+ }
} catch (IOException e) {
e.printStackTrace();
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/20cd0920/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java
index 271eaf9..8b3403f 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java
@@ -407,17 +407,7 @@ public class QueryMasterTask extends CompositeService {
/////////////////////////////////////////////////
if (queryContext.hasOutputPath()) {
outputDir = queryContext.getOutputPath();
- if (queryContext.isOutputOverwrite()) {
- if (defaultFS.exists(outputDir.getParent())) {
- if (defaultFS.exists(outputDir)) {
- defaultFS.delete(outputDir, true);
- LOG.info("The output directory '" + outputDir + "' is cleaned.");
- }
- } else {
- defaultFS.mkdirs(outputDir.getParent());
- LOG.info("The output directory's parent '" + outputDir.getParent() + "' is created.");
- }
- } else {
+ if (!queryContext.isOutputOverwrite()) {
if (defaultFS.exists(outputDir)) {
throw new IOException("The output directory '" + outputDir + " already exists.");
}