You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ch...@apache.org on 2016/08/26 15:23:48 UTC
[1/2] incubator-carbondata git commit: fix the problem of hdfs lock
and move the lock file inside the table folder
Repository: incubator-carbondata
Updated Branches:
refs/heads/master 24d93b15a -> d702e8262
fix the problem of hdfs lock and move the lock file inside the table folder
fix the problem of hdfs lock and move the lock file inside the table folder
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/8e75531a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/8e75531a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/8e75531a
Branch: refs/heads/master
Commit: 8e75531a447e92ef145b75b7acc86732a0228706
Parents: 24d93b1
Author: Zhangshunyu <zh...@huawei.com>
Authored: Wed Aug 24 14:30:50 2016 +0800
Committer: chenliang613 <ch...@apache.org>
Committed: Fri Aug 26 23:22:08 2016 +0800
----------------------------------------------------------------------
.../carbondata/core/constants/CarbonCommonConstants.java | 8 ++++++++
.../org/apache/carbondata/lcm/locks/HdfsFileLock.java | 11 ++++++++++-
2 files changed, 18 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8e75531a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 21e5562..69d9c3c 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -142,6 +142,14 @@ public final class CarbonCommonConstants {
*/
public static final String LOAD_FOLDER = "Segment_";
/**
+ * HDFSURL_PREFIX
+ */
+ public static final String HDFSURL_PREFIX = "hdfs://";
+ /**
+ * FS_DEFAULT_FS
+ */
+ public static final String FS_DEFAULT_FS = "fs.defaultFS";
+ /**
* RESTructure Folder
*/
public static final String RESTRUCTRE_FOLDER = "RS_";
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8e75531a/processing/src/main/java/org/apache/carbondata/lcm/locks/HdfsFileLock.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/lcm/locks/HdfsFileLock.java b/processing/src/main/java/org/apache/carbondata/lcm/locks/HdfsFileLock.java
index a992614..9770d7e 100644
--- a/processing/src/main/java/org/apache/carbondata/lcm/locks/HdfsFileLock.java
+++ b/processing/src/main/java/org/apache/carbondata/lcm/locks/HdfsFileLock.java
@@ -28,6 +28,8 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
import org.apache.carbondata.core.util.CarbonProperties;
+import org.apache.hadoop.conf.Configuration;
+
/**
* This class is used to handle the HDFS File locking.
* This is acheived using the concept of acquiring the data out stream using Append option.
@@ -46,8 +48,15 @@ public class HdfsFileLock extends AbstractCarbonLock {
public static String tmpPath;
static {
- tmpPath = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.HDFS_TEMP_LOCATION,
+ Configuration conf = new Configuration(true);
+ String hdfsPath = conf.get(CarbonCommonConstants.FS_DEFAULT_FS);
+ // By default, we put the hdfs lock meta file for one table inside this table's store folder.
+ // If can not get the STORE_LOCATION, then use hadoop.tmp.dir .
+ tmpPath = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.STORE_LOCATION,
System.getProperty(CarbonCommonConstants.HDFS_TEMP_LOCATION));
+ if (!tmpPath.startsWith(CarbonCommonConstants.HDFSURL_PREFIX)) {
+ tmpPath = hdfsPath + tmpPath;
+ }
}
/**
[2/2] incubator-carbondata git commit: [CARBONDATA-174]Fix the
problem of hdfs lock and move the lock file inside the table folder This
closes #89
Posted by ch...@apache.org.
[CARBONDATA-174]Fix the problem of hdfs lock and move the lock file inside the table folder This closes #89
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/d702e826
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/d702e826
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/d702e826
Branch: refs/heads/master
Commit: d702e826268f40e5064d26b972e69f4215976fd3
Parents: 24d93b1 8e75531
Author: chenliang613 <ch...@apache.org>
Authored: Fri Aug 26 23:23:29 2016 +0800
Committer: chenliang613 <ch...@apache.org>
Committed: Fri Aug 26 23:23:29 2016 +0800
----------------------------------------------------------------------
.../carbondata/core/constants/CarbonCommonConstants.java | 8 ++++++++
.../org/apache/carbondata/lcm/locks/HdfsFileLock.java | 11 ++++++++++-
2 files changed, 18 insertions(+), 1 deletion(-)
----------------------------------------------------------------------