You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2018/11/21 17:59:39 UTC
[12/50] [abbrv] carbondata git commit: [HOTFIX] change log level for
data loading
[HOTFIX] change log level for data loading
In current data loading, many log meant for debugging purpose is logged as INFO log, in order to reduce the entry of them, In this PR they are changed to DEBUG level.
This closes #2911
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/30f1dbdb
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/30f1dbdb
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/30f1dbdb
Branch: refs/heads/branch-1.5
Commit: 30f1dbdb9941b3163d9ea0fb1772dad091f5d27f
Parents: 60ba7bc
Author: Jacky Li <ja...@qq.com>
Authored: Fri Nov 9 14:49:15 2018 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Wed Nov 21 22:40:11 2018 +0530
----------------------------------------------------------------------
.../core/metadata/schema/table/TableInfo.java | 9 +++--
.../apache/carbondata/core/util/CarbonUtil.java | 39 +++++++++++---------
.../loading/AbstractDataLoadProcessorStep.java | 4 +-
.../processing/loading/DataLoadExecutor.java | 2 -
.../CarbonRowDataWriterProcessorStepImpl.java | 12 +++---
.../steps/DataWriterProcessorStepImpl.java | 12 +++---
.../store/CarbonFactDataHandlerColumnar.java | 35 ++++++++++++------
.../store/writer/AbstractFactDataWriter.java | 2 +-
.../writer/v3/CarbonFactDataWriterImplV3.java | 7 +++-
.../util/CarbonDataProcessorUtil.java | 1 -
10 files changed, 73 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/30f1dbdb/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java
index b3e9e7e..3e50586 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java
@@ -258,9 +258,12 @@ public class TableInfo implements Serializable, Writable {
}
if (null == tableBlockSize) {
tableBlockSize = CarbonCommonConstants.BLOCK_SIZE_DEFAULT_VAL;
- LOGGER.info("Table block size not specified for " + getTableUniqueName()
- + ". Therefore considering the default value "
- + CarbonCommonConstants.BLOCK_SIZE_DEFAULT_VAL + " MB");
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug(
+ "Table block size not specified for " + getTableUniqueName() +
+ ". Therefore considering the default value " +
+ CarbonCommonConstants.BLOCK_SIZE_DEFAULT_VAL + " MB");
+ }
}
return Integer.parseInt(tableBlockSize);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/30f1dbdb/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 1840ba0..2fa6260 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -2474,7 +2474,7 @@ public final class CarbonUtil {
lockAcquired = carbonLock.lockWithRetries();
}
if (lockAcquired) {
- LOGGER.info("Acquired lock for table for table status updation");
+ LOGGER.debug("Acquired lock for table for table status updation");
String metadataPath = carbonTable.getMetadataPath();
LoadMetadataDetails[] loadMetadataDetails =
SegmentStatusManager.readLoadMetadata(metadataPath);
@@ -2488,7 +2488,7 @@ public final class CarbonUtil {
// If it is old segment, need to calculate data size and index size again
if (null == dsize || null == isize) {
needUpdate = true;
- LOGGER.info("It is an old segment, need calculate data size and index size again");
+ LOGGER.debug("It is an old segment, need calculate data size and index size again");
HashMap<String, Long> map = CarbonUtil.getDataSizeAndIndexSize(
identifier.getTablePath(), loadMetadataDetail.getLoadName());
dsize = String.valueOf(map.get(CarbonCommonConstants.CARBON_TOTAL_DATA_SIZE));
@@ -2524,7 +2524,7 @@ public final class CarbonUtil {
}
} finally {
if (carbonLock.unlock()) {
- LOGGER.info("Table unlocked successfully after table status updation");
+ LOGGER.debug("Table unlocked successfully after table status updation");
} else {
LOGGER.error("Unable to unlock Table lock for table during table status updation");
}
@@ -2727,7 +2727,8 @@ public final class CarbonUtil {
String carbonDataDirectoryPath, long fileSizeInBytes)
throws CarbonDataWriterException {
long copyStartTime = System.currentTimeMillis();
- LOGGER.info("Copying " + localFilePath + " --> " + carbonDataDirectoryPath);
+ LOGGER.info(String.format("Copying %s to %s, operation id %d", localFilePath,
+ carbonDataDirectoryPath, copyStartTime));
try {
CarbonFile localCarbonFile =
FileFactory.getCarbonFile(localFilePath, FileFactory.getFileType(localFilePath));
@@ -2740,9 +2741,8 @@ public final class CarbonUtil {
throw new CarbonDataWriterException(
"Problem while copying file from local store to carbon store", e);
}
- LOGGER.info(
- "Total copy time (ms) to copy file " + localFilePath + " is " + (System.currentTimeMillis()
- - copyStartTime));
+ LOGGER.info(String.format("Total copy time is %d ms, operation id %d",
+ System.currentTimeMillis() - copyStartTime, copyStartTime));
}
/**
@@ -2797,10 +2797,12 @@ public final class CarbonUtil {
String readableBlockSize = ByteUtil.convertByteToReadable(blockSize);
String readableFileSize = ByteUtil.convertByteToReadable(fileSize);
String readableMaxSize = ByteUtil.convertByteToReadable(maxSize);
- LOGGER.info(
- "The configured block size is " + readableBlockSize + ", the actual carbon file size is "
- + readableFileSize + ", choose the max value " + readableMaxSize
- + " as the block size on HDFS");
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug(
+ "The configured block size is " + readableBlockSize + ", the actual carbon file size is "
+ + readableFileSize + ", choose the max value " + readableMaxSize
+ + " as the block size on HDFS");
+ }
return maxSize;
}
@@ -3117,10 +3119,11 @@ public final class CarbonUtil {
}
}
if (islocalDictEnabled) {
- LOGGER.info("Local dictionary is enabled for table: " + carbonTable.getTableUniqueName());
- LOGGER.info(
- "Local dictionary threshold for table: " + carbonTable.getTableUniqueName() + " is: "
- + carbonTable.getLocalDictionaryThreshold());
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("Local dictionary is enabled for table: " + carbonTable.getTableUniqueName());
+ LOGGER.debug(String.format("Local dictionary threshold for table %s is %d",
+ carbonTable.getTableUniqueName(), carbonTable.getLocalDictionaryThreshold()));
+ }
Iterator<Map.Entry<String, LocalDictionaryGenerator>> iterator =
columnLocalDictGenMap.entrySet().iterator();
StringBuilder stringBuilder = new StringBuilder();
@@ -3129,8 +3132,10 @@ public final class CarbonUtil {
stringBuilder.append(next.getKey());
stringBuilder.append(',');
}
- LOGGER.info("Local dictionary will be generated for the columns:" + stringBuilder.toString()
- + " for table: " + carbonTable.getTableUniqueName());
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug(String.format("Local dictionary will be generated for the columns: %s for"
+ + " table %s", stringBuilder.toString(), carbonTable.getTableUniqueName()));
+ }
}
return columnLocalDictGenMap;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/30f1dbdb/processing/src/main/java/org/apache/carbondata/processing/loading/AbstractDataLoadProcessorStep.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/AbstractDataLoadProcessorStep.java b/processing/src/main/java/org/apache/carbondata/processing/loading/AbstractDataLoadProcessorStep.java
index b93f21d..c3b587c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/AbstractDataLoadProcessorStep.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/AbstractDataLoadProcessorStep.java
@@ -72,13 +72,13 @@ public abstract class AbstractDataLoadProcessorStep {
* @throws IOException
*/
public void initialize() throws IOException {
- if (LOGGER.isInfoEnabled()) {
+ if (LOGGER.isDebugEnabled()) {
// This thread prints the rows processed in each step for every 10 seconds.
new Thread() {
@Override public void run() {
while (!closed) {
try {
- LOGGER.info("Rows processed in step " + getStepName() + " : " + rowCounter.get());
+ LOGGER.debug("Rows processed in step " + getStepName() + " : " + rowCounter.get());
Thread.sleep(10000);
} catch (InterruptedException e) {
//ignore
http://git-wip-us.apache.org/repos/asf/carbondata/blob/30f1dbdb/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadExecutor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadExecutor.java b/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadExecutor.java
index 6550afe..11841f9 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadExecutor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadExecutor.java
@@ -53,8 +53,6 @@ public class DataLoadExecutor {
// check and remove any bad record key from bad record entry logger static map
if (CarbonBadRecordUtil.hasBadRecord(loadModel)) {
LOGGER.error("Data Load is partially success for table " + loadModel.getTableName());
- } else {
- LOGGER.info("Data loading is successful for table " + loadModel.getTableName());
}
} catch (CarbonDataLoadingException e) {
if (e instanceof BadRecordFoundException) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/30f1dbdb/processing/src/main/java/org/apache/carbondata/processing/loading/steps/CarbonRowDataWriterProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/CarbonRowDataWriterProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/CarbonRowDataWriterProcessorStepImpl.java
index d4d48f9..0d38f5c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/CarbonRowDataWriterProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/CarbonRowDataWriterProcessorStepImpl.java
@@ -200,11 +200,13 @@ public class CarbonRowDataWriterProcessorStepImpl extends AbstractDataLoadProces
exception = new CarbonDataWriterException(
"Failed for table: " + tableName + " in finishing data handler", e);
}
- LOGGER.info("Record Processed For table: " + tableName);
- String logMessage =
- "Finished Carbon DataWriterProcessorStepImpl: Read: " + readCounter[iteratorIndex]
- + ": Write: " + readCounter[iteratorIndex];
- LOGGER.info(logMessage);
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("Record Processed For table: " + tableName);
+ String logMessage =
+ "Finished Carbon DataWriterProcessorStepImpl: Read: " + readCounter[iteratorIndex] +
+ ": Write: " + readCounter[iteratorIndex];
+ LOGGER.debug(logMessage);
+ }
CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordTotalRecords(rowCounter.get());
try {
processingComplete(dataHandler);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/30f1dbdb/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterProcessorStepImpl.java
index 2dc3275..6899886 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterProcessorStepImpl.java
@@ -194,11 +194,11 @@ public class DataWriterProcessorStepImpl extends AbstractDataLoadProcessorStep {
configuration.getTableIdentifier().getCarbonTableIdentifier();
String tableName = tableIdentifier.getTableName();
dataHandler.finish();
- LOGGER.info("Record Processed For table: " + tableName);
- String logMessage =
- "Finished Carbon DataWriterProcessorStepImpl: Read: " + readCounter + ": Write: "
- + rowCounter.get();
- LOGGER.info(logMessage);
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("Record Processed For table: " + tableName);
+ LOGGER.debug("Finished Carbon DataWriterProcessorStepImpl: Read: " + readCounter +
+ ": Write: " + rowCounter.get());
+ }
CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordTotalRecords(rowCounter.get());
processingComplete(dataHandler);
CarbonTimeStatisticsFactory.getLoadStatisticsInstance()
@@ -235,7 +235,7 @@ public class DataWriterProcessorStepImpl extends AbstractDataLoadProcessorStep {
super.close();
if (listener != null) {
try {
- LOGGER.info("closing all the DataMap writers registered to DataMap writer listener");
+ LOGGER.debug("closing all the DataMap writers registered to DataMap writer listener");
listener.finish();
} catch (IOException e) {
LOGGER.error("error while closing the datamap writers", e);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/30f1dbdb/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
index 67c4463..c58f8d8 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
@@ -134,7 +134,9 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
}
}
- LOGGER.info("Columns considered as NoInverted Index are " + noInvertedIdxCol.toString());
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("Columns considered as NoInverted Index are " + noInvertedIdxCol.toString());
+ }
}
private void initParameters(CarbonFactDataHandlerModel model) {
@@ -146,7 +148,7 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
.getBucketId()));
producerExecutorServiceTaskList =
new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- LOGGER.info("Initializing writer executors");
+ LOGGER.debug("Initializing writer executors");
consumerExecutorService = Executors.newFixedThreadPool(1, new CarbonThreadFactory(
"ConsumerPool_" + System.nanoTime() + ":" + model.getTableName() + ", range: " + model
.getBucketId()));
@@ -210,7 +212,10 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
blockletProcessingCount.incrementAndGet();
// set the entry count to zero
processedDataCount += entryCount;
- LOGGER.info("Total Number Of records added to store: " + processedDataCount);
+
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("Total Number Of records added to store: " + processedDataCount);
+ }
dataRows = new ArrayList<>(this.pageSize);
this.entryCount = 0;
} catch (InterruptedException e) {
@@ -256,7 +261,7 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
}
if (SnappyCompressor.MAX_BYTE_TO_COMPRESS -
(varcharColumnSizeInByte[i] + dataRows.size() * 4) < (2 << 20)) {
- LOGGER.info("Limited by varchar column, page size is " + dataRows.size());
+ LOGGER.debug("Limited by varchar column, page size is " + dataRows.size());
// re-init for next page
varcharColumnSizeInByte = new int[model.getVarcharDimIdxInNoDict().size()];
return true;
@@ -284,7 +289,9 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
tablePage.encode();
- LOGGER.info("Number Of records processed: " + dataRows.size());
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("Number Of records processed: " + dataRows.size());
+ }
return tablePage;
}
@@ -302,14 +309,18 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
if (producerExecutorService.isShutdown()) {
return;
}
- LOGGER.info("Started Finish Operation");
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("Started Finish Operation");
+ }
try {
semaphore.acquire();
producerExecutorServiceTaskList.add(producerExecutorService
.submit(new Producer(tablePageList, dataRows, ++writerTaskSequenceCounter, true)));
blockletProcessingCount.incrementAndGet();
processedDataCount += entryCount;
- LOGGER.info("Total Number Of records added to store: " + processedDataCount);
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("Total Number Of records added to store: " + processedDataCount);
+ }
closeWriterExecutionService(producerExecutorService);
processWriteTaskSubmitList(producerExecutorServiceTaskList);
processingComplete = true;
@@ -377,7 +388,9 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
consumerExecutorService.shutdownNow();
processWriteTaskSubmitList(consumerExecutorServiceTaskList);
this.dataWriter.writeFooter();
- LOGGER.info("All blocklets have been finished writing");
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("All blocklets have been finished writing");
+ }
// close all the open stream for both the files
this.dataWriter.closeWriter();
}
@@ -402,12 +415,12 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
pageSize :
CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT;
}
- LOGGER.info("Number of rows per column page is configured as pageSize = " + pageSize);
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("Number of rows per column page is configured as pageSize = " + pageSize);
+ }
dataRows = new ArrayList<>(this.pageSize);
if (model.getVarcharDimIdxInNoDict().size() > 0) {
- LOGGER.info("Number of rows per column blocklet is constrained by pageSize and actual size " +
- "of long string column(s)");
varcharColumnSizeInByte = new int[model.getVarcharDimIdxInNoDict().size()];
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/30f1dbdb/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
index 64084e8..31d3b68 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
@@ -282,7 +282,7 @@ public abstract class AbstractFactDataWriter implements CarbonFactDataWriter {
.add(executorService.submit(new CompleteHdfsBackendThread(carbonDataFileTempPath)));
}
} catch (IOException e) {
- LOGGER.error("Failed to delete carbondata file from temp location" + e.getMessage());
+ LOGGER.error(e);
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/30f1dbdb/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
index 7256c1e..48993c5 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
@@ -144,8 +144,11 @@ public class CarbonFactDataWriterImplV3 extends AbstractFactDataWriter {
addPageData(tablePage);
}
- LOGGER.info("Number of Pages for blocklet is: " + blockletDataHolder.getNumberOfPagesAdded()
- + " :Rows Added: " + blockletDataHolder.getTotalRows());
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("Number of Pages for blocklet is: " +
+ blockletDataHolder.getNumberOfPagesAdded() +
+ " :Rows Added: " + blockletDataHolder.getTotalRows());
+ }
// write the data
writeBlockletToFile();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/30f1dbdb/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
index 044e4e8..b34ed13 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
@@ -522,7 +522,6 @@ public final class CarbonDataProcessorUtil {
configuration.getDataLoadProperty(CarbonCommonConstants.LOAD_SORT_SCOPE)
.toString());
}
- LOGGER.info("sort scope is set to " + sortScope);
} catch (Exception e) {
sortScope = SortScopeOptions.getSortScope(CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT);
LOGGER.warn("Exception occured while resolving sort scope. " +