You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/04/07 09:55:22 UTC
[19/49] incubator-carbondata git commit: fixed issue with meta lock
not getting deleted for rename table
fixed issue with meta lock not getting deleted for rename table
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/7ae1cd8f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/7ae1cd8f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/7ae1cd8f
Branch: refs/heads/12-dev
Commit: 7ae1cd8fe77a391cb5f5addec4e19fbff5b06820
Parents: 9554971
Author: kunal642 <ku...@knoldus.in>
Authored: Fri Mar 31 16:03:17 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Apr 6 15:11:22 2017 +0530
----------------------------------------------------------------------
.../core/locks/AbstractCarbonLock.java | 9 +++++
.../carbondata/core/locks/HdfsFileLock.java | 36 +++++++++-----------
.../carbondata/core/locks/ICarbonLock.java | 8 +++++
.../carbondata/core/locks/LocalFileLock.java | 5 ++-
.../execution/command/AlterTableCommands.scala | 15 ++++++--
.../org/apache/spark/util/AlterTableUtil.scala | 1 +
.../AlterTableValidationTestCase.scala | 8 +++++
7 files changed, 60 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7ae1cd8f/core/src/main/java/org/apache/carbondata/core/locks/AbstractCarbonLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/locks/AbstractCarbonLock.java b/core/src/main/java/org/apache/carbondata/core/locks/AbstractCarbonLock.java
index eeb04d5..e927a7e 100644
--- a/core/src/main/java/org/apache/carbondata/core/locks/AbstractCarbonLock.java
+++ b/core/src/main/java/org/apache/carbondata/core/locks/AbstractCarbonLock.java
@@ -18,6 +18,7 @@
package org.apache.carbondata.core.locks;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.util.CarbonProperties;
/**
@@ -72,4 +73,12 @@ public abstract class AbstractCarbonLock implements ICarbonLock {
}
+ public boolean releaseLockManually(String lockFile) {
+ try {
+ return FileFactory.deleteFile(lockFile, FileFactory.getFileType(lockFile));
+ } catch (Exception e) {
+ return false;
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7ae1cd8f/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java b/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java
index aed73b3..985ced1 100644
--- a/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java
+++ b/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.util.CarbonProperties;
@@ -102,34 +103,31 @@ public class HdfsFileLock extends AbstractCarbonLock {
*/
@Override
public boolean unlock() {
+ boolean status = false;
if (null != dataOutputStream) {
try {
dataOutputStream.close();
+ status = true;
} catch (IOException e) {
- try {
- if (!FileFactory.isFileExist(location, FileFactory.getFileType(location))) {
- return true;
- }
- } catch (IOException e1) {
- LOGGER.error("Exception in isFileExist of the lock file " + e1.getMessage());
- }
- LOGGER.error("Exception in unlocking of the lock file " + e.getMessage());
- return false;
+ status = false;
} finally {
- try {
- if (FileFactory.isFileExist(location, FileFactory.getFileType(location))) {
- if (FileFactory.getCarbonFile(location, FileFactory.getFileType(location)).delete()) {
- LOGGER.info("Deleted the lock file " + location);
- } else {
- LOGGER.error("Not able to delete the lock file " + location);
- }
+ CarbonFile carbonFile =
+ FileFactory.getCarbonFile(location, FileFactory.getFileType(location));
+ if (carbonFile.exists()) {
+ if (carbonFile.delete()) {
+ LOGGER.info("Deleted the lock file " + location);
+ } else {
+ LOGGER.error("Not able to delete the lock file " + location);
+ status = false;
}
- } catch (IOException e) {
- LOGGER.error("Exception in isFileExist of the lock file " + e.getMessage());
+ } else {
+ LOGGER.error("Not able to delete the lock file because "
+ + "it is not existed in location " + location);
+ status = false;
}
}
}
- return true;
+ return status;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7ae1cd8f/core/src/main/java/org/apache/carbondata/core/locks/ICarbonLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/locks/ICarbonLock.java b/core/src/main/java/org/apache/carbondata/core/locks/ICarbonLock.java
index 51c577a..27862fd 100644
--- a/core/src/main/java/org/apache/carbondata/core/locks/ICarbonLock.java
+++ b/core/src/main/java/org/apache/carbondata/core/locks/ICarbonLock.java
@@ -36,4 +36,12 @@ public interface ICarbonLock {
*/
boolean lockWithRetries();
+ /**
+ * This method will delete the lock file at the specified location.
+ *
+ * @param lockFile The path of the lock file.
+ * @return True if the lock file is deleted, false otherwise.
+ */
+ boolean releaseLockManually(String lockFile);
+
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7ae1cd8f/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
index 616c9ed..2802127 100644
--- a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
+++ b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
@@ -28,6 +28,7 @@ import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.util.CarbonProperties;
/**
* This class handles the file locking in the local file system.
@@ -70,7 +71,8 @@ public class LocalFileLock extends AbstractCarbonLock {
LogServiceFactory.getLogService(LocalFileLock.class.getName());
static {
- tmpPath = System.getProperty("java.io.tmpdir");
+ tmpPath = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.STORE_LOCATION,
+ System.getProperty("java.io.tmpdir"));
}
/**
@@ -151,6 +153,7 @@ public class LocalFileLock extends AbstractCarbonLock {
LOGGER.info("Successfully deleted the lock file " + lockFilePath);
} else {
LOGGER.error("Not able to delete the lock file " + lockFilePath);
+ status = false;
}
} catch (IOException e) {
LOGGER.error(e.getMessage());
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7ae1cd8f/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/AlterTableCommands.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/AlterTableCommands.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/AlterTableCommands.scala
index 93a5912..88ca4af 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/AlterTableCommands.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/AlterTableCommands.scala
@@ -33,7 +33,7 @@ import org.apache.carbondata.core.metadata.{CarbonMetadata, CarbonTableIdentifie
import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl
import org.apache.carbondata.core.metadata.encoder.Encoding
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn
-import org.apache.carbondata.core.util.CarbonUtil
+import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
import org.apache.carbondata.core.util.path.CarbonStorePath
import org.apache.carbondata.format.{ColumnSchema, SchemaEvolutionEntry, TableInfo}
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
@@ -204,7 +204,18 @@ private[sql] case class AlterTableRenameTable(alterTableRenameModel: AlterTableR
if (carbonLock.unlock()) {
LOGGER.info("Lock released successfully")
} else {
- LOGGER.error("Unable to release lock during rename table")
+ val storeLocation = CarbonProperties.getInstance
+ .getProperty(CarbonCommonConstants.STORE_LOCATION,
+ System.getProperty("java.io.tmpdir"))
+ val lockFilePath = storeLocation + CarbonCommonConstants.FILE_SEPARATOR +
+ oldDatabaseName + CarbonCommonConstants.FILE_SEPARATOR + newTableName +
+ CarbonCommonConstants.FILE_SEPARATOR +
+ LockUsage.METADATA_LOCK
+ if(carbonLock.releaseLockManually(lockFilePath)) {
+ LOGGER.info("Lock released successfully")
+ } else {
+ LOGGER.error("Unable to release lock during rename table")
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7ae1cd8f/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala b/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala
index 0611e99..243eeb6 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala
@@ -30,6 +30,7 @@ import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.format.{SchemaEvolutionEntry, TableInfo}
object AlterTableUtil {
+
def validateTableAndAcquireLock(dbName: String, tableName: String, LOGGER: LogService)
(sparkSession: SparkSession): ICarbonLock = {
val relation =
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7ae1cd8f/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala
index bd34913..914136c 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala
@@ -381,11 +381,19 @@ class AlterTableValidationTestCase extends QueryTest with BeforeAndAfterAll {
sql("drop database testdb")
}
+ test("test to check if the lock file is successfully deleted") {
+ sql("create table lock_check(id int, name string) stored by 'carbondata'")
+ sql("alter table lock_check rename to lock_rename")
+ assert(!new File(s"${ CarbonCommonConstants.STORE_LOCATION } + /default/lock_rename/meta.lock")
+ .exists())
+ }
+
override def afterAll {
sql("DROP TABLE IF EXISTS restructure")
sql("DROP TABLE IF EXISTS restructure_new")
sql("DROP TABLE IF EXISTS restructure_test")
sql("DROP TABLE IF EXISTS restructure_bad")
sql("DROP TABLE IF EXISTS restructure_badnew")
+ sql("DROP TABLE IF EXISTS lock_rename")
}
}