You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by qi...@apache.org on 2018/03/23 14:03:49 UTC
carbondata git commit: [CARBONDATA-2220] Reduce unnecessary audit log
Repository: carbondata
Updated Branches:
refs/heads/master de52da40f -> 873c3ded0
[CARBONDATA-2220] Reduce unnecessary audit log
This closes #2020
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/873c3ded
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/873c3ded
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/873c3ded
Branch: refs/heads/master
Commit: 873c3ded04738fa9cbef9a5f0d37902fc758a88b
Parents: de52da4
Author: Jacky Li <ja...@qq.com>
Authored: Fri Mar 2 15:40:27 2018 +0800
Committer: QiangCai <qi...@qq.com>
Committed: Fri Mar 23 22:02:42 2018 +0800
----------------------------------------------------------------------
.../generator/TableDictionaryGenerator.java | 2 +-
.../server/NonSecureDictionaryServer.java | 2 +-
.../NonSecureDictionaryServerHandler.java | 2 +-
.../carbondata/core/scan/filter/FilterUtil.java | 4 +-
.../core/util/CarbonLoadStatisticsImpl.java | 50 ++++++++++----------
.../carbondata/core/util/CarbonProperties.java | 6 ++-
.../carbondata/examples/CompareTest.scala | 4 +-
.../management/CarbonInsertIntoCommand.scala | 2 +-
.../CarbonAlterTableDropPartitionCommand.scala | 1 -
.../CarbonAlterTableSplitPartitionCommand.scala | 1 -
.../spark/sql/hive/CarbonFileMetastore.scala | 2 +-
.../spark/sql/hive/CarbonHiveMetadataUtil.scala | 2 +-
.../apache/spark/sql/hive/CarbonMetaStore.scala | 4 +-
.../holder/UnsafeFinalMergePageHolder.java | 2 +-
.../unsafe/holder/UnsafeInmemoryHolder.java | 2 +-
.../holder/UnsafeInmemoryMergeHolder.java | 2 +-
.../holder/UnsafeSortTempFileChunkHolder.java | 2 +-
.../util/CarbonDataProcessorUtil.java | 15 ++++--
.../carbondata/streaming/StreamHandoffRDD.scala | 2 +-
19 files changed, 55 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/873c3ded/core/src/main/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGenerator.java b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGenerator.java
index 905d2fa..5db13b6 100644
--- a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGenerator.java
@@ -84,7 +84,7 @@ public class TableDictionaryGenerator
} catch (InterruptedException e) {
LOGGER.error("Error loading the dictionary: " + e.getMessage());
}
- LOGGER.audit("Total time taken to write dictionary file is: " +
+ LOGGER.info("Total time taken to write dictionary file is: " +
(System.currentTimeMillis() - start));
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/873c3ded/core/src/main/java/org/apache/carbondata/core/dictionary/server/NonSecureDictionaryServer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/server/NonSecureDictionaryServer.java b/core/src/main/java/org/apache/carbondata/core/dictionary/server/NonSecureDictionaryServer.java
index daba470..95f3d69 100644
--- a/core/src/main/java/org/apache/carbondata/core/dictionary/server/NonSecureDictionaryServer.java
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/server/NonSecureDictionaryServer.java
@@ -113,7 +113,7 @@ public class NonSecureDictionaryServer extends AbstractDictionaryServer
new InetSocketAddress(newPort) :
new InetSocketAddress(hostToBind, newPort);
bootstrap.bind(address).sync();
- LOGGER.audit("Dictionary Server started, Time spent " + (System.currentTimeMillis() - start)
+ LOGGER.info("Dictionary Server started, Time spent " + (System.currentTimeMillis() - start)
+ " Listening on port " + newPort);
this.port = newPort;
this.host = hostToBind;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/873c3ded/core/src/main/java/org/apache/carbondata/core/dictionary/server/NonSecureDictionaryServerHandler.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/server/NonSecureDictionaryServerHandler.java b/core/src/main/java/org/apache/carbondata/core/dictionary/server/NonSecureDictionaryServerHandler.java
index dc3d078..82efe80 100644
--- a/core/src/main/java/org/apache/carbondata/core/dictionary/server/NonSecureDictionaryServerHandler.java
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/server/NonSecureDictionaryServerHandler.java
@@ -48,7 +48,7 @@ import io.netty.channel.ChannelInboundHandlerAdapter;
* @throws Exception
*/
public void channelActive(ChannelHandlerContext ctx) throws Exception {
- LOGGER.audit("Connected " + ctx);
+ LOGGER.info("Connected " + ctx);
super.channelActive(ctx);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/873c3ded/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
index 0b72e77..a397355 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
@@ -777,7 +777,7 @@ public final class FilterUtil {
columnFilterInfo.setFilterList(filterValuesList);
}
} catch (FilterIllegalMemberException e) {
- LOGGER.audit(e.getMessage());
+ LOGGER.error(e.getMessage());
}
return columnFilterInfo;
}
@@ -817,7 +817,7 @@ public final class FilterUtil {
}
}
} catch (FilterIllegalMemberException e) {
- LOGGER.audit(e.getMessage());
+ LOGGER.error(e.getMessage());
}
if (null == defaultValues) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/873c3ded/core/src/main/java/org/apache/carbondata/core/util/CarbonLoadStatisticsImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonLoadStatisticsImpl.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonLoadStatisticsImpl.java
index c9fc8ba..9c6ab95 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonLoadStatisticsImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonLoadStatisticsImpl.java
@@ -311,52 +311,52 @@ public class CarbonLoadStatisticsImpl implements LoadStatistics {
//Print the statistics information
private void printDicGenStatisticsInfo() {
double loadCsvfilesToDfTime = getLoadCsvfilesToDfTime();
- LOGGER.audit("STAGE 1 ->Load csv to DataFrame and generate" +
+ LOGGER.info("STAGE 1 ->Load csv to DataFrame and generate" +
" block distinct values: " + loadCsvfilesToDfTime + "(s)");
double dicShuffleAndWriteFileTotalTime = getDicShuffleAndWriteFileTotalTime();
- LOGGER.audit("STAGE 2 ->Global dict shuffle and write dict file: " +
+ LOGGER.info("STAGE 2 ->Global dict shuffle and write dict file: " +
+ dicShuffleAndWriteFileTotalTime + "(s)");
}
private void printLruCacheLoadTimeInfo() {
- LOGGER.audit("STAGE 3 ->LRU cache load: " + getLruCacheLoadTime() + "(s)");
+ LOGGER.info("STAGE 3 ->LRU cache load: " + getLruCacheLoadTime() + "(s)");
}
private void printDictionaryValuesGenStatisticsInfo(String partitionID) {
double dictionaryValuesTotalTime = getDictionaryValuesTotalTime(partitionID);
- LOGGER.audit("STAGE 4 ->Total cost of gen dictionary values, sort and write to temp files: "
+ LOGGER.info("STAGE 4 ->Total cost of gen dictionary values, sort and write to temp files: "
+ dictionaryValuesTotalTime + "(s)");
double csvInputStepTime = getCsvInputStepTime(partitionID);
double generatingDictionaryValuesTime = getGeneratingDictionaryValuesTime(partitionID);
- LOGGER.audit("STAGE 4.1 -> |_read csv file: " + csvInputStepTime + "(s)");
- LOGGER.audit("STAGE 4.2 -> |_transform to surrogate key: "
+ LOGGER.info("STAGE 4.1 -> |_read csv file: " + csvInputStepTime + "(s)");
+ LOGGER.info("STAGE 4.2 -> |_transform to surrogate key: "
+ generatingDictionaryValuesTime + "(s)");
}
private void printSortRowsStepStatisticsInfo(String partitionID) {
double sortRowsStepTotalTime = getSortRowsStepTotalTime(partitionID);
- LOGGER.audit("STAGE 4.3 -> |_sort rows and write to temp file: "
+ LOGGER.info("STAGE 4.3 -> |_sort rows and write to temp file: "
+ sortRowsStepTotalTime + "(s)");
}
private void printGenMdkStatisticsInfo(String partitionID) {
double dictionaryValue2MdkAdd2FileTime = getDictionaryValue2MdkAdd2FileTime(partitionID);
- LOGGER.audit("STAGE 5 ->Transform to MDK, compress and write fact files: "
+ LOGGER.info("STAGE 5 ->Transform to MDK, compress and write fact files: "
+ dictionaryValue2MdkAdd2FileTime + "(s)");
}
//Print the node blocks information
private void printHostBlockMapInfo() {
- LOGGER.audit("========== BLOCK_INFO ==========");
+ LOGGER.info("========== BLOCK_INFO ==========");
if (getHostBlockMap().size() > 0) {
for (String host: getHostBlockMap().keySet()) {
- LOGGER.audit("BLOCK_INFO ->Node host: " + host);
- LOGGER.audit("BLOCK_INFO ->The block count in this node: " + getHostBlockMap().get(host));
+ LOGGER.info("BLOCK_INFO ->Node host: " + host);
+ LOGGER.info("BLOCK_INFO ->The block count in this node: " + getHostBlockMap().get(host));
}
} else if (getPartitionBlockMap().size() > 0) {
for (String parID: getPartitionBlockMap().keySet()) {
- LOGGER.audit("BLOCK_INFO ->Partition ID: " + parID);
- LOGGER.audit("BLOCK_INFO ->The block count in this partition: " +
+ LOGGER.info("BLOCK_INFO ->Partition ID: " + parID);
+ LOGGER.info("BLOCK_INFO ->The block count in this partition: " +
getPartitionBlockMap().get(parID));
}
}
@@ -364,21 +364,21 @@ public class CarbonLoadStatisticsImpl implements LoadStatistics {
//Print the speed information
private void printLoadSpeedInfo(String partitionID) {
- LOGGER.audit("===============Load_Speed_Info===============");
- LOGGER.audit("Total Num of Records Processed: " + getTotalRecords());
- LOGGER.audit("Total Time Cost: " + getTotalTime(partitionID) + "(s)");
- LOGGER.audit("Total Load Speed: " + getLoadSpeed() + "records/s");
- LOGGER.audit("Generate Dictionaries Speed: " + getGenDicSpeed() + "records/s");
- LOGGER.audit("Read CSV Speed: " + getReadCSVSpeed(partitionID) + " records/s");
- LOGGER.audit("Generate Surrogate Key Speed: " + getGenSurKeySpeed(partitionID) + " records/s");
- LOGGER.audit("Sort Key/Write Temp Files Speed: " + getSortKeySpeed(partitionID) + " records/s");
- LOGGER.audit("MDK Step Speed: " + getMDKSpeed(partitionID) + " records/s");
- LOGGER.audit("=============================================");
+ LOGGER.info("===============Load_Speed_Info===============");
+ LOGGER.info("Total Num of Records Processed: " + getTotalRecords());
+ LOGGER.info("Total Time Cost: " + getTotalTime(partitionID) + "(s)");
+ LOGGER.info("Total Load Speed: " + getLoadSpeed() + "records/s");
+ LOGGER.info("Generate Dictionaries Speed: " + getGenDicSpeed() + "records/s");
+ LOGGER.info("Read CSV Speed: " + getReadCSVSpeed(partitionID) + " records/s");
+ LOGGER.info("Generate Surrogate Key Speed: " + getGenSurKeySpeed(partitionID) + " records/s");
+ LOGGER.info("Sort Key/Write Temp Files Speed: " + getSortKeySpeed(partitionID) + " records/s");
+ LOGGER.info("MDK Step Speed: " + getMDKSpeed(partitionID) + " records/s");
+ LOGGER.info("=============================================");
}
public void printStatisticsInfo(String partitionID) {
try {
- LOGGER.audit("========== TIME_STATISTICS PartitionID: " + partitionID + "==========");
+ LOGGER.info("========== TIME_STATISTICS PartitionID: " + partitionID + "==========");
printDicGenStatisticsInfo();
printLruCacheLoadTimeInfo();
printDictionaryValuesGenStatisticsInfo(partitionID);
@@ -387,7 +387,7 @@ public class CarbonLoadStatisticsImpl implements LoadStatistics {
printHostBlockMapInfo();
printLoadSpeedInfo(partitionID);
} catch (Exception e) {
- LOGGER.audit("Can't print Statistics Information");
+ LOGGER.error("Can't print Statistics Information");
} finally {
resetLoadStatistics();
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/873c3ded/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
index acc266c..5ac359b 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
@@ -947,8 +947,10 @@ public final class CarbonProperties {
public int getNumberOfCores() {
int numberOfCores;
try {
- numberOfCores = Integer.parseInt(CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.NUM_CORES_LOADING));
+ numberOfCores = Integer.parseInt(
+ CarbonProperties.getInstance().getProperty(
+ CarbonCommonConstants.NUM_CORES_LOADING,
+ CarbonCommonConstants.NUM_CORES_DEFAULT_VAL));
} catch (NumberFormatException exc) {
LOGGER.error("Configured value for property " + CarbonCommonConstants.NUM_CORES_LOADING
+ " is wrong. Falling back to the default value "
http://git-wip-us.apache.org/repos/asf/carbondata/blob/873c3ded/examples/spark2/src/main/scala/org/apache/carbondata/examples/CompareTest.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CompareTest.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CompareTest.scala
index 64ccca9..d27b1c4 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CompareTest.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CompareTest.scala
@@ -268,16 +268,14 @@ object CompareTest {
private def loadCarbonTable(spark: SparkSession, input: DataFrame, tableName: String): Double = {
CarbonProperties.getInstance().addProperty(
CarbonCommonConstants.CARBON_DATA_FILE_VERSION,
- "3"
+ "V3"
)
spark.sql(s"drop table if exists $tableName")
time {
input.write
.format("carbondata")
.option("tableName", tableName)
- .option("tempCSV", "false")
.option("single_pass", "true")
- .option("dictionary_exclude", "id") // id is high cardinality column
.option("table_blocksize", "32")
.mode(SaveMode.Overwrite)
.save()
http://git-wip-us.apache.org/repos/asf/carbondata/blob/873c3ded/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoCommand.scala
index 86d6759..702f954 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoCommand.scala
@@ -52,7 +52,7 @@ case class CarbonInsertIntoCommand(
isPersistEnabledUserValue.equalsIgnoreCase("true") || containsLimit(child)
val df =
if (isPersistRequired) {
- LOGGER.audit("Persist enabled for Insert operation")
+ LOGGER.info("Persist enabled for Insert operation")
Dataset.ofRows(sparkSession, child)
.persist(StorageLevel.MEMORY_AND_DISK)
} else {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/873c3ded/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropPartitionCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropPartitionCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropPartitionCommand.scala
index 705455f..9c2835e 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropPartitionCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropPartitionCommand.scala
@@ -158,7 +158,6 @@ case class CarbonAlterTableDropPartitionCommand(
CacheProvider.getInstance().dropAllCache()
AlterTableUtil.releaseLocks(locks)
LOGGER.info("Locks released after alter table drop partition action.")
- LOGGER.audit("Locks released after alter table drop partition action.")
}
LOGGER.info(s"Alter table drop partition is successful for table $dbName.$tableName")
LOGGER.audit(s"Alter table drop partition is successful for table $dbName.$tableName")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/873c3ded/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableSplitPartitionCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableSplitPartitionCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableSplitPartitionCommand.scala
index 99691d2..4b89296 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableSplitPartitionCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableSplitPartitionCommand.scala
@@ -167,7 +167,6 @@ case class CarbonAlterTableSplitPartitionCommand(
CacheProvider.getInstance().dropAllCache()
val LOGGER: LogService = LogServiceFactory.getLogService(this.getClass.getName)
LOGGER.info("Locks released after alter table add/split partition action.")
- LOGGER.audit("Locks released after alter table add/split partition action.")
if (success) {
LOGGER.info(s"Alter table add/split partition is successful for table $dbName.$tableName")
LOGGER.audit(s"Alter table add/split partition is successful for table $dbName.$tableName")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/873c3ded/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
index b3438a4..52bae1a 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
@@ -487,7 +487,7 @@ class CarbonFileMetastore extends CarbonMetaStore {
private def touchSchemaFileSystemTime(): Long = {
val (timestampFile, timestampFileType) = getTimestampFileAndType()
if (!FileFactory.isFileExist(timestampFile, timestampFileType)) {
- LOGGER.audit(s"Creating timestamp file for $timestampFile")
+ LOGGER.info(s"Creating timestamp file for $timestampFile")
FileFactory
.createNewFile(timestampFile,
timestampFileType,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/873c3ded/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetadataUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetadataUtil.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetadataUtil.scala
index d67ae6b..dfa9904 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetadataUtil.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetadataUtil.scala
@@ -46,7 +46,7 @@ object CarbonHiveMetadataUtil {
sparkSession.sessionState.catalog.dropTable(tabelIdentifier, true, false)
} catch {
case e: Exception =>
- LOGGER.audit(
+ LOGGER.error(
s"Error While deleting the table $databaseName.$tableName during drop carbon table" +
e.getMessage)
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/873c3ded/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala
index 3f1ebbc..9127f14 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala
@@ -184,10 +184,10 @@ object CarbonMetaStoreFactory {
def createCarbonMetaStore(conf: RuntimeConfig): CarbonMetaStore = {
val readSchemaFromHiveMetaStore = readSchemaFromHive(conf)
if (readSchemaFromHiveMetaStore) {
- LOGGER.audit("Hive based carbon metastore is enabled")
+ LOGGER.info("Hive based carbon metastore is enabled")
new CarbonHiveMetaStore()
} else {
- LOGGER.audit("File based carbon metastore is enabled")
+ LOGGER.info("File based carbon metastore is enabled")
new CarbonFileMetastore()
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/873c3ded/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeFinalMergePageHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeFinalMergePageHolder.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeFinalMergePageHolder.java
index a776db1..c966ff2 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeFinalMergePageHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeFinalMergePageHolder.java
@@ -49,7 +49,7 @@ public class UnsafeFinalMergePageHolder implements SortTempChunkHolder {
this.mergedAddresses = merger.getMergedAddresses();
this.rowPageIndexes = merger.getRowPageIndexes();
this.rowPages = merger.getUnsafeCarbonRowPages();
- LOGGER.audit("Processing unsafe inmemory rows page with size : " + actualSize);
+ LOGGER.info("Processing unsafe inmemory rows page with size : " + actualSize);
this.comparator = new IntermediateSortTempRowComparator(noDictSortColumnMapping);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/873c3ded/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeInmemoryHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeInmemoryHolder.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeInmemoryHolder.java
index cbcbbae..8b4b550 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeInmemoryHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeInmemoryHolder.java
@@ -43,7 +43,7 @@ public class UnsafeInmemoryHolder implements SortTempChunkHolder {
public UnsafeInmemoryHolder(UnsafeCarbonRowPage rowPage) {
this.actualSize = rowPage.getBuffer().getActualSize();
this.rowPage = rowPage;
- LOGGER.audit("Processing unsafe inmemory rows page with size : " + actualSize);
+ LOGGER.info("Processing unsafe inmemory rows page with size : " + actualSize);
this.comparator = new IntermediateSortTempRowComparator(
rowPage.getTableFieldStat().getIsSortColNoDictFlags());
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/873c3ded/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeInmemoryMergeHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeInmemoryMergeHolder.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeInmemoryMergeHolder.java
index 3b9d8d7..f8689d9 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeInmemoryMergeHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeInmemoryMergeHolder.java
@@ -48,7 +48,7 @@ public class UnsafeInmemoryMergeHolder implements Comparable<UnsafeInmemoryMerge
public UnsafeInmemoryMergeHolder(UnsafeCarbonRowPage rowPage, byte index) {
this.actualSize = rowPage.getBuffer().getActualSize();
this.rowPage = rowPage;
- LOGGER.audit("Processing unsafe inmemory rows page with size : " + actualSize);
+ LOGGER.info("Processing unsafe inmemory rows page with size : " + actualSize);
this.comparator = new UnsafeRowComparator(rowPage);
this.baseObject = rowPage.getDataBlock().getBaseObject();
currentRow = new UnsafeCarbonRowForMerge();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/873c3ded/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
index 527452a..86b7ac8 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
@@ -131,7 +131,7 @@ public class UnsafeSortTempFileChunkHolder implements SortTempChunkHolder {
stream = FileFactory.getDataInputStream(tempFile.getPath(), FileFactory.FileType.LOCAL,
readBufferSize, compressorName);
this.entryCount = stream.readInt();
- LOGGER.audit("Processing unsafe mode file rows with size : " + entryCount);
+ LOGGER.info("Processing unsafe mode file rows with size : " + entryCount);
if (prefetch) {
new DataFetcher(false).call();
totalRecordFetch += currentBuffer.length;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/873c3ded/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
index ba2b0c2..392ad59 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
@@ -110,8 +110,13 @@ public final class CarbonDataProcessorUtil {
*/
public static void createLocations(String[] locations) {
for (String loc : locations) {
- if (!new File(loc).mkdirs()) {
- LOGGER.warn("Error occurs while creating dirs: " + loc);
+ File dir = new File(loc);
+ if (dir.exists()) {
+ LOGGER.warn("dir already exists, skip dir creation: " + loc);
+ } else {
+ if (!dir.mkdirs()) {
+ LOGGER.error("Error occurs while creating dir: " + loc);
+ }
}
}
}
@@ -433,7 +438,7 @@ public final class CarbonDataProcessorUtil {
configuration.getDataLoadProperty(CarbonCommonConstants.LOAD_SORT_SCOPE)
.toString());
}
- LOGGER.warn("sort scope is set to " + sortScope);
+ LOGGER.info("sort scope is set to " + sortScope);
} catch (Exception e) {
sortScope = SortScopeOptions.getSortScope(CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT);
LOGGER.warn("Exception occured while resolving sort scope. " +
@@ -453,7 +458,7 @@ public final class CarbonDataProcessorUtil {
} else {
sortScope = SortScopeOptions.getSortScope(sortScopeString);
}
- LOGGER.warn("sort scope is set to " + sortScope);
+ LOGGER.info("sort scope is set to " + sortScope);
} catch (Exception e) {
sortScope = SortScopeOptions.getSortScope(CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT);
LOGGER.warn("Exception occured while resolving sort scope. " +
@@ -481,7 +486,7 @@ public final class CarbonDataProcessorUtil {
configuration.getDataLoadProperty(CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB)
.toString());
}
- LOGGER.warn("batch sort size is set to " + batchSortSizeInMb);
+ LOGGER.info("batch sort size is set to " + batchSortSizeInMb);
} catch (Exception e) {
batchSortSizeInMb = 0;
LOGGER.warn("Exception occured while resolving batch sort size. " +
http://git-wip-us.apache.org/repos/asf/carbondata/blob/873c3ded/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
index 47ef5f2..0a95c24 100644
--- a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
+++ b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
@@ -351,7 +351,7 @@ object StreamHandoffRDD {
LOGGER.info("********clean up done**********")
LOGGER.audit(s"Handoff is failed for " +
s"${ carbonLoadModel.getDatabaseName }.${ carbonLoadModel.getTableName }")
- LOGGER.warn("Cannot write load metadata file as handoff failed")
+ LOGGER.error("Cannot write load metadata file as handoff failed")
throw new Exception(errorMessage)
}