You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2017/11/17 19:38:25 UTC
hive git commit: HIVE-14560: Support exchange partition between s3
and hdfs tables (Abdullah Yousufi via Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master c1d9b2021 -> f87315a79
HIVE-14560: Support exchange partition between s3 and hdfs tables (Abdullah Yousufi via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f87315a7
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f87315a7
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f87315a7
Branch: refs/heads/master
Commit: f87315a79296f4715beb9d45be291b74addae940
Parents: c1d9b20
Author: Abdullah Yousufi <ab...@cloudera.com>
Authored: Fri Aug 19 10:45:15 2016 -0700
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Fri Nov 17 11:37:59 2017 -0800
----------------------------------------------------------------------
.../apache/hadoop/hive/metastore/Warehouse.java | 5 +--
.../hadoop/hive/metastore/utils/FileUtils.java | 36 +++++++++++++-------
2 files changed, 27 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/f87315a7/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/Warehouse.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/Warehouse.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/Warehouse.java
index 4672e55..421d1be 100755
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/Warehouse.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/Warehouse.java
@@ -206,8 +206,9 @@ public class Warehouse {
// location, we should make a copy of the files to cmroot instead of moving it.
cm.recycle(sourcePath, RecycleType.COPY, true);
}
- FileSystem fs = getFs(sourcePath);
- return FileUtils.rename(fs, sourcePath, destPath);
+ FileSystem srcFs = getFs(sourcePath);
+ FileSystem destFs = getFs(destPath);
+ return FileUtils.rename(srcFs, destFs, sourcePath, destPath);
} catch (Exception ex) {
MetaStoreUtils.logAndThrowMetaException(ex);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/f87315a7/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java
index 2dac899..0543274 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java
@@ -153,23 +153,35 @@ public class FileUtils {
* Rename a file. Unlike {@link FileSystem#rename(Path, Path)}, if the destPath already exists
* and is a directory, this will NOT move the sourcePath into it. It will throw an IOException
* instead.
- * @param fs file system paths are on
+ * @param srcfs file system src paths are on
+ * @param destfs file system dest paths are on
* @param sourcePath source file or directory to move
* @param destPath destination file name. This must be a file and not an existing directory.
* @return result of fs.rename.
* @throws IOException if fs.rename throws it, or if destPath already exists.
*/
- public static boolean rename(FileSystem fs, Path sourcePath, Path destPath) throws IOException {
- LOG.info("Renaming " + sourcePath + " to " + destPath);
-
- // If destPath directory exists, rename call will move the sourcePath
- // into destPath without failing. So check it before renaming.
- if (fs.exists(destPath)) {
- throw new IOException("Cannot rename the source path. The destination "
- + "path already exists.");
- }
- return fs.rename(sourcePath, destPath);
- }
+ public static boolean rename(FileSystem srcFs, FileSystem destFs, Path srcPath,
+ Path destPath) throws IOException {
+ LOG.info("Renaming " + srcPath + " to " + destPath);
+
+ // If destPath directory exists, rename call will move the srcPath
+ // into destPath without failing. So check it before renaming.
+ if(destFs.exists(destPath)) {
+ throw new IOException("Cannot rename the source path. The destination "
+ + "path already exists.");
+ }
+
+ if (equalsFileSystem(srcFs, destFs)) {
+ //just rename the directory
+ return srcFs.rename(srcPath, destPath);
+ } else {
+ Configuration conf = new Configuration();
+ return copy(srcFs, srcPath, destFs, destPath,
+ true, // delete source
+ false, // overwrite destination
+ conf);
+ }
+ }
// NOTE: This is for generating the internal path name for partitions. Users
// should always use the MetaStore API to get the path name for a partition.