You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2016/08/01 10:05:06 UTC
[08/47] incubator-carbondata git commit: [CARBONDATA-84] Change lock
framework to accept locklocation (#844)
[CARBONDATA-84] Change lock framework to accept locklocation (#844)
1.get dictionary lock location from CarbonTablePath
2.Change lock framework to accept lock location
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/584402e8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/584402e8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/584402e8
Branch: refs/heads/master
Commit: 584402e8008d64b57233dc5be2a13a2739e32c61
Parents: 07116d2
Author: ashokblend <as...@gmail.com>
Authored: Sat Jul 23 11:06:21 2016 +0530
Committer: Venkata Ramana G <g....@gmail.com>
Committed: Sat Jul 23 11:06:21 2016 +0530
----------------------------------------------------------------------
.../core/carbon/path/CarbonStorePath.java | 2 +-
.../core/carbon/path/CarbonTablePath.java | 18 +++++++++++++++-
.../core/locks/CarbonLockFactory.java | 22 ++++++++++++++++++++
.../org/carbondata/core/locks/HdfsFileLock.java | 16 ++++++++++----
.../carbondata/core/locks/LocalFileLock.java | 16 ++++++++++----
.../carbondata/core/locks/ZooKeeperLocking.java | 18 ++++++++--------
.../spark/rdd/CarbonGlobalDictionaryRDD.scala | 9 ++++++--
7 files changed, 80 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/584402e8/core/src/main/java/org/carbondata/core/carbon/path/CarbonStorePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/carbon/path/CarbonStorePath.java b/core/src/main/java/org/carbondata/core/carbon/path/CarbonStorePath.java
index c393f43..eac98e6 100644
--- a/core/src/main/java/org/carbondata/core/carbon/path/CarbonStorePath.java
+++ b/core/src/main/java/org/carbondata/core/carbon/path/CarbonStorePath.java
@@ -41,7 +41,7 @@ public class CarbonStorePath extends Path {
*/
public static CarbonTablePath getCarbonTablePath(String storePath,
CarbonTableIdentifier tableIdentifier) {
- CarbonTablePath carbonTablePath = new CarbonTablePath(
+ CarbonTablePath carbonTablePath = new CarbonTablePath(tableIdentifier,
storePath + File.separator + tableIdentifier.getDatabaseName() + File.separator
+ tableIdentifier.getTableName());
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/584402e8/core/src/main/java/org/carbondata/core/carbon/path/CarbonTablePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/carbon/path/CarbonTablePath.java b/core/src/main/java/org/carbondata/core/carbon/path/CarbonTablePath.java
index 9ac8802..bd2c472 100644
--- a/core/src/main/java/org/carbondata/core/carbon/path/CarbonTablePath.java
+++ b/core/src/main/java/org/carbondata/core/carbon/path/CarbonTablePath.java
@@ -20,6 +20,7 @@ package org.carbondata.core.carbon.path;
import java.io.File;
+import org.carbondata.core.carbon.CarbonTableIdentifier;
import org.carbondata.core.constants.CarbonCommonConstants;
import org.carbondata.core.datastorage.store.filesystem.CarbonFile;
import org.carbondata.core.datastorage.store.filesystem.CarbonFileFilter;
@@ -50,9 +51,16 @@ public class CarbonTablePath extends Path {
protected static final String INDEX_FILE_EXT = ".carbonindex";
protected String tablePath;
+ protected CarbonTableIdentifier carbonTableIdentifier;
- public CarbonTablePath(String tablePathString) {
+ /**
+ *
+ * @param carbonTableIdentifier
+ * @param tablePathString
+ */
+ public CarbonTablePath(CarbonTableIdentifier carbonTableIdentifier, String tablePathString) {
super(tablePathString);
+ this.carbonTableIdentifier = carbonTableIdentifier;
this.tablePath = tablePathString;
}
@@ -127,6 +135,14 @@ public class CarbonTablePath extends Path {
}
/**
+ * @return it return relative directory
+ */
+ public String getRelativeDictionaryDirectory() {
+ return carbonTableIdentifier.getDatabaseName() + File.separator + carbonTableIdentifier
+ .getTableName();
+ }
+
+ /**
* This method will return the metadata directory location for a table
*
* @return
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/584402e8/core/src/main/java/org/carbondata/core/locks/CarbonLockFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/locks/CarbonLockFactory.java b/core/src/main/java/org/carbondata/core/locks/CarbonLockFactory.java
index dcf2c2c..8616c68 100644
--- a/core/src/main/java/org/carbondata/core/locks/CarbonLockFactory.java
+++ b/core/src/main/java/org/carbondata/core/locks/CarbonLockFactory.java
@@ -62,6 +62,28 @@ public class CarbonLockFactory {
}
/**
+ *
+ * @param locFileLocation
+ * @param lockFile
+ * @return carbon lock
+ */
+ public static ICarbonLock getCarbonLockObj(String locFileLocation, String lockFile) {
+ switch (lockTypeConfigured.toUpperCase()) {
+ case CarbonCommonConstants.CARBON_LOCK_TYPE_LOCAL:
+ return new LocalFileLock(locFileLocation, lockFile);
+
+ case CarbonCommonConstants.CARBON_LOCK_TYPE_ZOOKEEPER:
+ return new ZooKeeperLocking(locFileLocation, lockFile);
+
+ case CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS:
+ return new HdfsFileLock(locFileLocation, lockFile);
+
+ default:
+ throw new UnsupportedOperationException("Not supported the lock type");
+ }
+ }
+
+ /**
* This method will set the zookeeper status whether zookeeper to be used for locking or not.
*/
private static void updateZooKeeperLockingStatus() {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/584402e8/core/src/main/java/org/carbondata/core/locks/HdfsFileLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/locks/HdfsFileLock.java b/core/src/main/java/org/carbondata/core/locks/HdfsFileLock.java
index e1fdd73..e8f44ed 100644
--- a/core/src/main/java/org/carbondata/core/locks/HdfsFileLock.java
+++ b/core/src/main/java/org/carbondata/core/locks/HdfsFileLock.java
@@ -45,14 +45,22 @@ public class HdfsFileLock extends AbstractCarbonLock {
}
/**
+ * @param lockFileLocation
+ * @param lockFile
+ */
+ public HdfsFileLock(String lockFileLocation, String lockFile) {
+ this.location = tmpPath + CarbonCommonConstants.FILE_SEPARATOR + lockFileLocation
+ + CarbonCommonConstants.FILE_SEPARATOR + lockFile;
+ initRetry();
+ }
+
+ /**
* @param tableIdentifier
* @param lockFile
*/
public HdfsFileLock(CarbonTableIdentifier tableIdentifier, String lockFile) {
- this.location =
- tmpPath + CarbonCommonConstants.FILE_SEPARATOR + tableIdentifier.getDatabaseName()
- + CarbonCommonConstants.FILE_SEPARATOR + tableIdentifier.getTableName()
- + CarbonCommonConstants.FILE_SEPARATOR + lockFile;
+ this(tableIdentifier.getDatabaseName() + CarbonCommonConstants.FILE_SEPARATOR + tableIdentifier
+ .getTableName(), lockFile);
initRetry();
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/584402e8/core/src/main/java/org/carbondata/core/locks/LocalFileLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/locks/LocalFileLock.java b/core/src/main/java/org/carbondata/core/locks/LocalFileLock.java
index 8194bfd..1120091 100644
--- a/core/src/main/java/org/carbondata/core/locks/LocalFileLock.java
+++ b/core/src/main/java/org/carbondata/core/locks/LocalFileLock.java
@@ -73,14 +73,22 @@ public class LocalFileLock extends AbstractCarbonLock {
}
/**
+ * @param lockFileLocation
+ * @param lockFile
+ */
+ public LocalFileLock(String lockFileLocation, String lockFile) {
+ this.location = tmpPath + CarbonCommonConstants.FILE_SEPARATOR + lockFileLocation;
+ this.lockFile = lockFile;
+ initRetry();
+ }
+
+ /**
* @param tableIdentifier
* @param lockFile
*/
public LocalFileLock(CarbonTableIdentifier tableIdentifier, String lockFile) {
- this.location =
- tmpPath + CarbonCommonConstants.FILE_SEPARATOR + tableIdentifier.getDatabaseName()
- + CarbonCommonConstants.FILE_SEPARATOR + tableIdentifier.getTableName();
- this.lockFile = lockFile;
+ this(tableIdentifier.getDatabaseName() + CarbonCommonConstants.FILE_SEPARATOR + tableIdentifier
+ .getTableName(), lockFile);
initRetry();
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/584402e8/core/src/main/java/org/carbondata/core/locks/ZooKeeperLocking.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/locks/ZooKeeperLocking.java b/core/src/main/java/org/carbondata/core/locks/ZooKeeperLocking.java
index 47349ad..e1db3c6 100644
--- a/core/src/main/java/org/carbondata/core/locks/ZooKeeperLocking.java
+++ b/core/src/main/java/org/carbondata/core/locks/ZooKeeperLocking.java
@@ -68,22 +68,22 @@ public class ZooKeeperLocking extends AbstractCarbonLock {
private String lockTypeFolder;
+ public ZooKeeperLocking(CarbonTableIdentifier tableIdentifier, String lockFile) {
+ this(tableIdentifier.getDatabaseName() + '.' + tableIdentifier.getTableName(), lockFile);
+ }
+
/**
- * @param tableIdentifier
+ * @param lockLocation
* @param lockFile
*/
- public ZooKeeperLocking(CarbonTableIdentifier tableIdentifier, String lockFile) {
+ public ZooKeeperLocking(String lockLocation, String lockFile) {
this.lockName = lockFile;
- this.tableIdFolder =
- zooKeeperLocation + CarbonCommonConstants.FILE_SEPARATOR + tableIdentifier.getDatabaseName()
- + '.' + tableIdentifier.getTableName();
+ this.tableIdFolder = zooKeeperLocation + CarbonCommonConstants.FILE_SEPARATOR + lockLocation;
zk = ZookeeperInit.getInstance().getZookeeper();
- this.lockTypeFolder =
- zooKeeperLocation + CarbonCommonConstants.FILE_SEPARATOR + tableIdentifier.getDatabaseName()
- + '.' + tableIdentifier.getTableName() + CarbonCommonConstants.FILE_SEPARATOR
- + lockFile;
+ this.lockTypeFolder = zooKeeperLocation + CarbonCommonConstants.FILE_SEPARATOR + lockLocation
+ + CarbonCommonConstants.FILE_SEPARATOR + lockFile;
try {
createBaseNode();
// if exists returns null then path doesnt exist. so creating.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/584402e8/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
index 9abcceb..e519030 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
@@ -30,6 +30,7 @@ import org.apache.spark._
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.Row
+import org.carbondata.common.factory.CarbonCommonFactory
import org.carbondata.common.logging.LogServiceFactory
import org.carbondata.core.carbon.{CarbonTableIdentifier, ColumnIdentifier}
import org.carbondata.core.carbon.metadata.datatype.DataType
@@ -295,8 +296,12 @@ class CarbonGlobalDictionaryGenerateRDD(
var dictionaryForDistinctValueLookUp: org.carbondata.core.cache.dictionary.Dictionary = _
var dictionaryForSortIndexWriting: org.carbondata.core.cache.dictionary.Dictionary = _
var dictionaryForDistinctValueLookUpCleared: Boolean = false
- val dictLock = CarbonLockFactory.getCarbonLockObj(model.table,
- model.columnIdentifier(split.index).getColumnId + LockUsage.LOCK)
+ val pathService = CarbonCommonFactory.getPathService
+ val carbonTablePath = pathService.getCarbonTablePath(model.columnIdentifier(split.index),
+ model.hdfsLocation, model.table)
+ val dictLock = CarbonLockFactory
+ .getCarbonLockObj(carbonTablePath.getRelativeDictionaryDirectory,
+ model.columnIdentifier(split.index).getColumnId + LockUsage.LOCK)
// generate distinct value list
try {
val t1 = System.currentTimeMillis