You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ku...@apache.org on 2020/05/07 13:18:46 UTC
[carbondata] branch master updated: [CARBONDATA-3792]Refactor
system folder location and removed unwanted property
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 5eff5dc [CARBONDATA-3792]Refactor system folder location and removed unwanted property
5eff5dc is described below
commit 5eff5dcb68851199f4982110cbdab4dc1fd6bee3
Author: akashrn5 <ak...@gmail.com>
AuthorDate: Mon May 4 20:34:40 2020 +0530
[CARBONDATA-3792]Refactor system folder location and removed unwanted property
Why is this PR needed?
Currently the carbon.system.folder.location is being used only in test code,
which is not required and since the system folder location is based on database
and that code is being duplicated.
What changes were proposed in this PR?
Refactor code to get the system folder location based on db
location without duplicate code and remove the carbon.system.folder.location
usage in all the test classes and remove unnecessary code.
This closes #3743
---
.../core/constants/CarbonCommonConstants.java | 6 ---
.../carbondata/core/util/CarbonProperties.java | 31 +++-------------
.../carbondata/core/util/path/CarbonTablePath.java | 2 +
.../apache/carbondata/core/view/MVProvider.java | 19 +++++-----
.../hadoop/ft/CarbonTableInputFormatTest.java | 2 -
.../hadoop/ft/CarbonTableOutputFormatTest.java | 2 -
.../carbondata/hive/Hive2CarbonExpressionTest.java | 2 -
.../PrestoAllDataTypeLocalDictTest.scala | 3 --
.../integrationtest/PrestoAllDataTypeTest.scala | 4 --
.../PrestoTestNonTransactionalTableFiles.scala | 3 --
.../apache/spark/sql/test/TestQueryExecutor.scala | 6 ---
.../org/apache/spark/sql/test/util/QueryTest.scala | 5 +--
.../bloom/BloomCoarseGrainIndexFunctionSuite.scala | 2 -
.../index/bloom/BloomCoarseGrainIndexSuite.scala | 1 -
.../index/lucene/LuceneFineGrainIndexSuite.scala | 1 -
...ncodingUnsafeColumnPageForComplexDataType.scala | 3 --
...ingUnsafeHeapColumnPageForComplexDataType.scala | 3 --
.../spark/testsuite/index/CGIndexTestCase.scala | 1 -
.../spark/testsuite/index/TestIndexStatus.scala | 2 -
.../org/apache/carbondata/sdk/file/TestUtil.java | 43 ----------------------
.../carbondata/sdk/file/ArrowCarbonReaderTest.java | 18 ---------
.../carbondata/sdk/file/AvroCarbonWriterTest.java | 23 ------------
.../carbondata/sdk/file/CSVCarbonWriterTest.java | 24 ------------
.../carbondata/sdk/file/CarbonReaderTest.java | 27 --------------
.../carbondata/store/LocalCarbonStoreTest.java | 22 -----------
25 files changed, 19 insertions(+), 236 deletions(-)
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 9d418d4..130eca1 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -352,12 +352,6 @@ public final class CarbonCommonConstants {
*/
public static final String CARBON_INVISIBLE_SEGMENTS_PRESERVE_COUNT_DEFAULT = "200";
- /**
- * System older location to store system level data like index schema and status files.
- */
- @CarbonProperty
- public static final String CARBON_SYSTEM_FOLDER_LOCATION = "carbon.system.folder.location";
-
@CarbonProperty
public static final String CARBON_INDEX_SCHEMA_STORAGE = "carbon.index.schema.storage";
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
index 1d8e6ae..a238877 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
@@ -36,6 +36,7 @@ import org.apache.carbondata.core.constants.CarbonV3DataFormatConstants;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.util.annotations.CarbonProperty;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
import static org.apache.carbondata.core.constants.CarbonCommonConstants.BLOCKLET_SIZE;
import static org.apache.carbondata.core.constants.CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION;
@@ -1585,33 +1586,11 @@ public final class CarbonProperties {
}
/**
- * Get the configured system folder location.
- * @return
- */
- public String getSystemFolderLocation() {
- return getSystemFolderLocation(null);
- }
-
- /**
- * Get the configured system folder location.
- * @return
+ * Get the system folder location based on database location.
*/
- public String getSystemFolderLocation(String databaseName) {
- String systemLocation = CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.CARBON_SYSTEM_FOLDER_LOCATION);
- if (systemLocation == null) {
- systemLocation = getStorePath();
- }
- if (systemLocation != null) {
- systemLocation = CarbonUtil.checkAndAppendFileSystemURIScheme(systemLocation);
- systemLocation = FileFactory.getUpdatedFilePath(systemLocation);
- }
- if (databaseName == null) {
- return systemLocation + CarbonCommonConstants.FILE_SEPARATOR + "_system";
- } else {
- return systemLocation + CarbonCommonConstants.FILE_SEPARATOR +
- databaseName + CarbonCommonConstants.FILE_SEPARATOR + "_system";
- }
+ public String getSystemFolderLocationPerDatabase(String databaseLocation) {
+ return databaseLocation + CarbonCommonConstants.FILE_SEPARATOR
+ + CarbonTablePath.SYSTEM_FOLDER_DIR;
}
/**
diff --git a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
index 7afc752..a862af0 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
@@ -62,6 +62,8 @@ public class CarbonTablePath {
public static final String SUCCESS_FILE_SUBFIX = ".success";
private static final String SNAPSHOT_FILE_NAME = "snapshot";
+ public static final String SYSTEM_FOLDER_DIR = "_system";
+
/**
* This class provides static utility only.
*/
diff --git a/core/src/main/java/org/apache/carbondata/core/view/MVProvider.java b/core/src/main/java/org/apache/carbondata/core/view/MVProvider.java
index 524c31f..6593299 100644
--- a/core/src/main/java/org/apache/carbondata/core/view/MVProvider.java
+++ b/core/src/main/java/org/apache/carbondata/core/view/MVProvider.java
@@ -185,11 +185,10 @@ public class MVProvider {
return Arrays.asList(statusDetails);
}
- private static ICarbonLock getStatusLock(String databaseName) {
- return CarbonLockFactory
- .getSystemLevelCarbonLockObj(
- CarbonProperties.getInstance().getSystemFolderLocation(databaseName),
- LockUsage.MATERIALIZED_VIEW_STATUS_LOCK);
+ private static ICarbonLock getStatusLock(String databaseLocation) {
+ return CarbonLockFactory.getSystemLevelCarbonLockObj(
+ CarbonProperties.getInstance().getSystemFolderLocationPerDatabase(databaseLocation),
+ LockUsage.MATERIALIZED_VIEW_STATUS_LOCK);
}
/**
@@ -223,7 +222,9 @@ public class MVProvider {
private void updateStatus(MVManager viewManager, String databaseName, List<MVSchema> schemaList,
MVStatus status) throws IOException {
- ICarbonLock carbonTableStatusLock = getStatusLock(databaseName);
+ String databaseLocation =
+ FileFactory.getCarbonFile(viewManager.getDatabaseLocation(databaseName)).getCanonicalPath();
+ ICarbonLock carbonTableStatusLock = getStatusLock(databaseLocation);
boolean locked = false;
try {
locked = carbonTableStatusLock.lockWithRetries();
@@ -379,10 +380,10 @@ public class MVProvider {
SchemaProvider(String databaseLocation) {
final String systemDirectory =
- databaseLocation + CarbonCommonConstants.FILE_SEPARATOR + "_system";
+ CarbonProperties.getInstance().getSystemFolderLocationPerDatabase(databaseLocation);
this.systemDirectory = systemDirectory;
- this.schemaIndexFilePath = systemDirectory + CarbonCommonConstants.FILE_SEPARATOR +
- "mv_schema_index";
+ this.schemaIndexFilePath =
+ systemDirectory + CarbonCommonConstants.FILE_SEPARATOR + "mv_schema_index";
}
void saveSchema(MVManager viewManager, MVSchema viewSchema)
diff --git a/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonTableInputFormatTest.java b/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonTableInputFormatTest.java
index 5fdea35..3863740 100644
--- a/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonTableInputFormatTest.java
+++ b/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonTableInputFormatTest.java
@@ -66,8 +66,6 @@ public class CarbonTableInputFormatTest {
CarbonProperties.getInstance().
addProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC, "/tmp/carbon/badrecords");
CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_SYSTEM_FOLDER_LOCATION, "/tmp/carbon/");
- CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_WRITTEN_BY_APPNAME, "CarbonTableInputFormatTest");
try {
creator = new StoreCreator(new File("target/store").getAbsolutePath(),
diff --git a/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonTableOutputFormatTest.java b/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonTableOutputFormatTest.java
index 347be39..e9f1b36 100644
--- a/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonTableOutputFormatTest.java
+++ b/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonTableOutputFormatTest.java
@@ -53,8 +53,6 @@ public class CarbonTableOutputFormatTest {
CarbonProperties.getInstance().
addProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC, "/tmp/carbon/badrecords");
CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_SYSTEM_FOLDER_LOCATION, "/tmp/carbon/");
- CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_WRITTEN_BY_APPNAME, "CarbonTableOutputFormatTest");
try {
carbonLoadModel = new StoreCreator(new File("target/store").getAbsolutePath(),
diff --git a/integration/hive/src/test/java/org/apache/carbondata/hive/Hive2CarbonExpressionTest.java b/integration/hive/src/test/java/org/apache/carbondata/hive/Hive2CarbonExpressionTest.java
index f130ece..f54cf2e 100644
--- a/integration/hive/src/test/java/org/apache/carbondata/hive/Hive2CarbonExpressionTest.java
+++ b/integration/hive/src/test/java/org/apache/carbondata/hive/Hive2CarbonExpressionTest.java
@@ -66,8 +66,6 @@ public class Hive2CarbonExpressionTest {
CarbonProperties.getInstance().
addProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC, "/tmp/carbon/badrecords");
CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_SYSTEM_FOLDER_LOCATION, "/tmp/carbon/");
- CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_WRITTEN_BY_APPNAME, "Hive2CarbonExpressionTest");
try {
creator = new StoreCreator(new File("target/store").getAbsolutePath(),
diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeLocalDictTest.scala b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeLocalDictTest.scala
index e901113..9652a92 100644
--- a/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeLocalDictTest.scala
+++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeLocalDictTest.scala
@@ -38,7 +38,6 @@ class PrestoAllDataTypeLocalDictTest extends FunSuiteLike with BeforeAndAfterAll
private val rootPath = new File(this.getClass.getResource("/").getPath
+ "../../../..").getCanonicalPath
private val storePath = s"$rootPath/integration/presto/target/store"
- private val systemPath = s"$rootPath/integration/presto/target/system"
private val prestoServer = new PrestoServer
// Table schema:
@@ -70,8 +69,6 @@ class PrestoAllDataTypeLocalDictTest extends FunSuiteLike with BeforeAndAfterAll
override def beforeAll: Unit = {
import org.apache.carbondata.presto.util.CarbonDataStoreCreator
- CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_SYSTEM_FOLDER_LOCATION,
- systemPath)
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_WRITTEN_BY_APPNAME,
"Presto")
val map = new util.HashMap[String, String]()
diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala
index 21c95de..b854363 100644
--- a/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala
+++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala
@@ -27,7 +27,6 @@ import org.scalatest.{BeforeAndAfterAll, FunSuiteLike}
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.impl.FileFactory
-import org.apache.carbondata.core.datastore.impl.FileFactory.FileType
import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
import org.apache.carbondata.presto.server.PrestoServer
@@ -40,7 +39,6 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
private val rootPath = new File(this.getClass.getResource("/").getPath
+ "../../../..").getCanonicalPath
private val storePath = s"$rootPath/integration/presto/target/store"
- private val systemPath = s"$rootPath/integration/presto/target/system"
private val prestoServer = new PrestoServer
// Table schema:
@@ -72,8 +70,6 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
override def beforeAll: Unit = {
import org.apache.carbondata.presto.util.CarbonDataStoreCreator
- CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_SYSTEM_FOLDER_LOCATION,
- systemPath)
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_WRITTEN_BY_APPNAME,
"Presto")
val map = new util.HashMap[String, String]()
diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoTestNonTransactionalTableFiles.scala b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoTestNonTransactionalTableFiles.scala
index 741d2a3..7a6f3b6 100644
--- a/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoTestNonTransactionalTableFiles.scala
+++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoTestNonTransactionalTableFiles.scala
@@ -43,14 +43,11 @@ class PrestoTestNonTransactionalTableFiles extends FunSuiteLike with BeforeAndAf
private val rootPath = new File(this.getClass.getResource("/").getPath
+ "../../../..").getCanonicalPath
private val storePath = s"$rootPath/integration/presto/target/store"
- private val systemPath = s"$rootPath/integration/presto/target/system"
private val writerPath = storePath + "/sdk_output/files"
private val prestoServer = new PrestoServer
private var varcharString = new String
override def beforeAll: Unit = {
- CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_SYSTEM_FOLDER_LOCATION,
- systemPath)
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_WRITTEN_BY_APPNAME,
"Presto")
val map = new util.HashMap[String, String]()
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/test/TestQueryExecutor.scala b/integration/spark/src/main/scala/org/apache/spark/sql/test/TestQueryExecutor.scala
index 9267bbd..af27b4d 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/test/TestQueryExecutor.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/test/TestQueryExecutor.scala
@@ -124,11 +124,6 @@ object TestQueryExecutor {
} else {
s"$target/bad_store"
}
- val systemFolderPath = if (hdfsUrl.startsWith("hdfs://")) {
- s"$hdfsUrl/systemfolder" + System.nanoTime()
- } else {
- s"$target/systemfolder"
- }
createDirectory(badStoreLocation)
val hiveresultpath = if (hdfsUrl.startsWith("hdfs://")) {
@@ -173,7 +168,6 @@ object TestQueryExecutor {
.addProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC, badStoreLocation)
.addProperty(CarbonCommonConstants.CARBON_MAX_DRIVER_LRU_CACHE_SIZE, "1024")
.addProperty(CarbonCommonConstants.CARBON_MAX_EXECUTOR_LRU_CACHE_SIZE, "1024")
- .addProperty(CarbonCommonConstants.CARBON_SYSTEM_FOLDER_LOCATION, systemFolderPath)
.addProperty(CarbonCommonConstants.CARBON_MINMAX_ALLOWED_BYTE_COUNT, "40")
private def lookupQueryExecutor: Class[_] = {
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/test/util/QueryTest.scala b/integration/spark/src/main/scala/org/apache/spark/sql/test/util/QueryTest.scala
index 700cd8e..4cc6cb1 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/test/util/QueryTest.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/test/util/QueryTest.scala
@@ -29,7 +29,7 @@ import org.apache.spark.sql.test.TestQueryExecutor
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.cache.CacheProvider
import org.apache.carbondata.core.constants.{CarbonCommonConstants, CarbonLoadOptionConstants}
-import org.apache.carbondata.core.util.{CarbonProperties, SessionParams, ThreadLocalSessionInfo}
+import org.apache.carbondata.core.util.{CarbonProperties, ThreadLocalSessionInfo}
@@ -166,8 +166,7 @@ class QueryTest extends PlanTest {
def setCarbonProperties(propertiesString: String): Unit = {
val properties = propertiesString.split(", ", -1)
- val exclude = Set("carbon.system.folder.location",
- "carbon.badRecords.location",
+ val exclude = Set("carbon.badRecords.location",
"carbon.storelocation")
properties.foreach { property =>
val entry = property.split("=")
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/index/bloom/BloomCoarseGrainIndexFunctionSuite.scala b/integration/spark/src/test/scala/org/apache/carbondata/index/bloom/BloomCoarseGrainIndexFunctionSuite.scala
index 8db25ae..cb7c7b3 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/index/bloom/BloomCoarseGrainIndexFunctionSuite.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/index/bloom/BloomCoarseGrainIndexFunctionSuite.scala
@@ -17,7 +17,6 @@
package org.apache.carbondata.index.bloom
-import java.io.File
import java.util.{Random, UUID}
import scala.collection.JavaConverters._
@@ -45,7 +44,6 @@ class BloomCoarseGrainIndexFunctionSuite extends QueryTest with BeforeAndAfterA
override protected def beforeAll(): Unit = {
deleteFile(bigFile)
- new File(CarbonProperties.getInstance().getSystemFolderLocation).delete()
createFile(bigFile, line = 2000)
sql(s"DROP TABLE IF EXISTS $normalTable")
sql(s"DROP TABLE IF EXISTS $bloomSampleTable")
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/index/bloom/BloomCoarseGrainIndexSuite.scala b/integration/spark/src/test/scala/org/apache/carbondata/index/bloom/BloomCoarseGrainIndexSuite.scala
index b5282cf..d9135d6 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/index/bloom/BloomCoarseGrainIndexSuite.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/index/bloom/BloomCoarseGrainIndexSuite.scala
@@ -41,7 +41,6 @@ class BloomCoarseGrainIndexSuite extends QueryTest with BeforeAndAfterAll with B
val indexName = "bloom_dm"
override protected def beforeAll(): Unit = {
- new File(CarbonProperties.getInstance().getSystemFolderLocation).delete()
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.ENABLE_QUERY_STATISTICS, "true")
createFile(bigFile, line = 50000)
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/index/lucene/LuceneFineGrainIndexSuite.scala b/integration/spark/src/test/scala/org/apache/carbondata/index/lucene/LuceneFineGrainIndexSuite.scala
index e64cd38..24b59b5 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/index/lucene/LuceneFineGrainIndexSuite.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/index/lucene/LuceneFineGrainIndexSuite.scala
@@ -45,7 +45,6 @@ class LuceneFineGrainIndexSuite extends QueryTest with BeforeAndAfterAll {
sql("drop database if exists lucene cascade")
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.ENABLE_QUERY_STATISTICS, "true")
- new File(CarbonProperties.getInstance().getSystemFolderLocation).delete()
LuceneFineGrainIndexSuite.createFile(file2)
sql("create database if not exists lucene")
CarbonProperties.getInstance()
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveEncodingUnsafeColumnPageForComplexDataType.scala b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveEncodingUnsafeColumnPageForComplexDataType.scala
index 74c3778..51f085d 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveEncodingUnsafeColumnPageForComplexDataType.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveEncodingUnsafeColumnPageForComplexDataType.scala
@@ -17,8 +17,6 @@
package org.apache.carbondata.spark.testsuite.dataload
-import java.io.File
-
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.integration.spark.testsuite.complexType.TestAdaptiveComplexType
@@ -35,7 +33,6 @@ class TestAdaptiveEncodingUnsafeColumnPageForComplexDataType
override def beforeAll(): Unit = {
- new File(CarbonProperties.getInstance().getSystemFolderLocation).delete()
sql("DROP TABLE IF EXISTS adaptive")
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.ENABLE_UNSAFE_COLUMN_PAGE,
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveEncodingUnsafeHeapColumnPageForComplexDataType.scala b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveEncodingUnsafeHeapColumnPageForComplexDataType.scala
index acf75c1..8b1f9c5 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveEncodingUnsafeHeapColumnPageForComplexDataType.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveEncodingUnsafeHeapColumnPageForComplexDataType.scala
@@ -17,8 +17,6 @@
package org.apache.carbondata.integration.spark.testsuite.complexType
-import java.io.File
-
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.BeforeAndAfterAll
@@ -35,7 +33,6 @@ class TestAdaptiveEncodingUnsafeHeapColumnPageForComplexDataType
override def beforeAll(): Unit = {
- new File(CarbonProperties.getInstance().getSystemFolderLocation).delete()
sql("DROP TABLE IF EXISTS adaptive")
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.ENABLE_UNSAFE_COLUMN_PAGE,
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/index/CGIndexTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/index/CGIndexTestCase.scala
index dc56419..40a9639 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/index/CGIndexTestCase.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/index/CGIndexTestCase.scala
@@ -361,7 +361,6 @@ class CGIndexWriter(
class CGIndexTestCase extends QueryTest with BeforeAndAfterAll {
val file2 = resourcesPath + "/compaction/fil2.csv"
- val systemFolderStoreLocation = CarbonProperties.getInstance().getSystemFolderLocation
override protected def beforeAll(): Unit = {
//n should be about 5000000 of reset if size is default 1024
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/index/TestIndexStatus.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/index/TestIndexStatus.scala
index 31ee149..832f9cf 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/index/TestIndexStatus.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/index/TestIndexStatus.scala
@@ -17,7 +17,6 @@
package org.apache.carbondata.spark.testsuite.index
-import java.io.File
import java.util
import scala.collection.JavaConverters._
@@ -45,7 +44,6 @@ class TestIndexStatus extends QueryTest with BeforeAndAfterAll {
val testData = s"$resourcesPath/sample.csv"
override def beforeAll: Unit = {
- new File(CarbonProperties.getInstance().getSystemFolderLocation).delete()
drop
}
diff --git a/sdk/sdk/src/main/java/org/apache/carbondata/sdk/file/TestUtil.java b/sdk/sdk/src/main/java/org/apache/carbondata/sdk/file/TestUtil.java
index bbc2f17..a4279b4 100644
--- a/sdk/sdk/src/main/java/org/apache/carbondata/sdk/file/TestUtil.java
+++ b/sdk/sdk/src/main/java/org/apache/carbondata/sdk/file/TestUtil.java
@@ -27,8 +27,6 @@ import java.io.InputStream;
import org.apache.carbondata.common.annotations.InterfaceAudience;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastore.impl.FileFactory;
-import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.avro.file.DataFileWriter;
import org.apache.avro.generic.GenericData;
@@ -139,45 +137,4 @@ public class TestUtil {
throw new RuntimeException("Test failed: dataFiles is empty");
}
}
-
- /**
- * verify whether the file exists
- * if delete the file success or file not exists, then return true; otherwise return false
- *
- * @return boolean
- */
- public static boolean cleanMdtFile() {
- String fileName = CarbonProperties.getInstance().getSystemFolderLocation()
- + CarbonCommonConstants.FILE_SEPARATOR + "index.mdtfile";
- try {
- if (FileFactory.isFileExist(fileName)) {
- File file = new File(fileName);
- return file.delete();
- } else {
- return true;
- }
- } catch (IOException e) {
- e.printStackTrace();
- return false;
- }
- }
-
- /**
- * verify whether the mdt file exists
- * if the file exists, then return true; otherwise return false
- *
- * @return boolean
- */
- public static boolean verifyMdtFile() {
- String fileName = CarbonProperties.getInstance().getSystemFolderLocation()
- + CarbonCommonConstants.FILE_SEPARATOR + "index.mdtfile";
- try {
- if (FileFactory.isFileExist(fileName)) {
- return true;
- }
- return false;
- } catch (IOException e) {
- throw new RuntimeException("IO exception:", e);
- }
- }
}
diff --git a/sdk/sdk/src/test/java/org/apache/carbondata/sdk/file/ArrowCarbonReaderTest.java b/sdk/sdk/src/test/java/org/apache/carbondata/sdk/file/ArrowCarbonReaderTest.java
index c610b10..cd2e519 100644
--- a/sdk/sdk/src/test/java/org/apache/carbondata/sdk/file/ArrowCarbonReaderTest.java
+++ b/sdk/sdk/src/test/java/org/apache/carbondata/sdk/file/ArrowCarbonReaderTest.java
@@ -39,29 +39,11 @@ import org.apache.arrow.vector.VectorLoader;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
import org.apache.commons.io.FileUtils;
-import org.junit.After;
import org.junit.Assert;
-import org.junit.Before;
import org.junit.Test;
public class ArrowCarbonReaderTest extends TestCase {
- @Before
- public void cleanFile() {
- assert (TestUtil.cleanMdtFile());
- }
-
- @After
- public void verifyDMFile() {
- assert (!TestUtil.verifyMdtFile());
- String path = "./testWriteFiles";
- try {
- FileUtils.deleteDirectory(new File(path));
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-
@Test
public void testArrowReader() {
String path = "./carbondata";
diff --git a/sdk/sdk/src/test/java/org/apache/carbondata/sdk/file/AvroCarbonWriterTest.java b/sdk/sdk/src/test/java/org/apache/carbondata/sdk/file/AvroCarbonWriterTest.java
index 6cbce82..76a7040 100644
--- a/sdk/sdk/src/test/java/org/apache/carbondata/sdk/file/AvroCarbonWriterTest.java
+++ b/sdk/sdk/src/test/java/org/apache/carbondata/sdk/file/AvroCarbonWriterTest.java
@@ -33,13 +33,10 @@ import org.apache.carbondata.core.metadata.datatype.Field;
import org.apache.carbondata.core.metadata.schema.SchemaReader;
import org.apache.carbondata.core.metadata.schema.table.TableInfo;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
-import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.avro.generic.GenericData;
import org.apache.commons.io.FileUtils;
-import org.junit.After;
import org.junit.Assert;
-import org.junit.Before;
import org.junit.Test;
import org.apache.avro.Schema;
@@ -47,26 +44,6 @@ import org.apache.avro.Schema;
public class AvroCarbonWriterTest {
private String path = "./AvroCarbonWriterSuiteWriteFiles";
- @Before
- public void cleanFile() {
- String path = null;
- try {
- path = new File(AvroCarbonWriterTest.class.getResource("/").getPath() + "../")
- .getCanonicalPath().replaceAll("\\\\", "/");
- } catch (IOException e) {
- assert (false);
- }
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_SYSTEM_FOLDER_LOCATION, path);
- assert (TestUtil.cleanMdtFile());
- }
-
- @After
- public void verifyDMFile() throws IOException {
- FileUtils.deleteDirectory(new File(path));
- assert (!TestUtil.verifyMdtFile());
- }
-
@Test
public void testWriteBasic() throws IOException {
FileUtils.deleteDirectory(new File(path));
diff --git a/sdk/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java b/sdk/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java
index b90d5fe..7b9c040 100644
--- a/sdk/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java
+++ b/sdk/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java
@@ -42,14 +42,11 @@ import org.apache.carbondata.core.metadata.schema.SchemaReader;
import org.apache.carbondata.core.metadata.schema.table.TableInfo;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.reader.CarbonFooterReaderV3;
-import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.format.FileFooter3;
import org.apache.commons.io.FileUtils;
-import org.junit.After;
import org.junit.Assert;
-import org.junit.Before;
import org.junit.Test;
import static org.apache.carbondata.sdk.file.utils.SDKUtil.readObjects;
@@ -59,27 +56,6 @@ import static org.apache.carbondata.sdk.file.utils.SDKUtil.readObjects;
*/
public class CSVCarbonWriterTest {
- @Before
- public void cleanFile() {
- String path = null;
- try {
- path = new File(CSVCarbonWriterTest.class.getResource("/").getPath() + "../")
- .getCanonicalPath().replaceAll("\\\\", "/");
- } catch (IOException e) {
- assert (false);
- }
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_SYSTEM_FOLDER_LOCATION, path)
- .addProperty(CarbonCommonConstants.DETAIL_QUERY_BATCH_SIZE,
- String.valueOf(CarbonCommonConstants.DETAIL_QUERY_BATCH_SIZE_DEFAULT));
- assert (TestUtil.cleanMdtFile());
- }
-
- @After
- public void verifyDMFile() {
- assert (!TestUtil.verifyMdtFile());
- }
-
@Test
public void testWriteFiles() throws IOException {
String path = "./testWriteFiles";
diff --git a/sdk/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java b/sdk/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
index 153be5e..d47d5b3 100644
--- a/sdk/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
+++ b/sdk/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
@@ -42,9 +42,7 @@ import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.log4j.Logger;
-import org.junit.After;
import org.junit.Assert;
-import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
@@ -67,22 +65,6 @@ import static org.apache.carbondata.core.scan.filter.FilterUtil.prepareOrExpress
public class CarbonReaderTest extends TestCase {
- @Before
- public void cleanFile() {
- assert (TestUtil.cleanMdtFile());
- }
-
- @After
- public void verifyDMFile() {
- assert (!TestUtil.verifyMdtFile());
- String path = "./testWriteFiles";
- try {
- FileUtils.deleteDirectory(new File(path));
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-
@Test
public void testWriteAndReadFiles() throws IOException, InterruptedException {
String path = "./testWriteFiles";
@@ -1061,15 +1043,6 @@ public class CarbonReaderTest extends TestCase {
@Override
public void setUp() {
carbonProperties = CarbonProperties.getInstance();
- String path = null;
- try {
- path = new File(CarbonReaderTest.class.getResource("/").getPath() + "../")
- .getCanonicalPath().replaceAll("\\\\", "/");
- } catch (IOException e) {
- assert (false);
- }
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_SYSTEM_FOLDER_LOCATION, path);
}
private static final Logger LOGGER =
diff --git a/sdk/sdk/src/test/java/org/apache/carbondata/store/LocalCarbonStoreTest.java b/sdk/sdk/src/test/java/org/apache/carbondata/store/LocalCarbonStoreTest.java
index eca96fd..c7efae5 100644
--- a/sdk/sdk/src/test/java/org/apache/carbondata/store/LocalCarbonStoreTest.java
+++ b/sdk/sdk/src/test/java/org/apache/carbondata/store/LocalCarbonStoreTest.java
@@ -21,39 +21,17 @@ import java.io.File;
import java.io.IOException;
import java.util.Iterator;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.row.CarbonRow;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.metadata.datatype.Field;
import org.apache.carbondata.sdk.file.Schema;
import org.apache.carbondata.sdk.file.TestUtil;
import org.apache.commons.io.FileUtils;
-import org.junit.After;
-import org.junit.Before;
import org.junit.Test;
public class LocalCarbonStoreTest {
- @Before
- public void cleanFile() {
- String path = null;
- try {
- path = new File(LocalCarbonStoreTest.class.getResource("/").getPath() + "../")
- .getCanonicalPath().replaceAll("\\\\", "/");
- } catch (IOException e) {
- assert (false);
- }
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_SYSTEM_FOLDER_LOCATION, path);
- assert (TestUtil.cleanMdtFile());
- }
-
- @After
- public void verifyDMFile() {
- assert (!TestUtil.verifyMdtFile());
- }
// TODO: complete this testcase
// Currently result rows are empty, because SDK is not writing table status file