You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by sn...@apache.org on 2018/12/01 00:30:20 UTC
[incubator-pinot] 01/01: Improve the logging for segment merge
command 1. add min start / max end time for input segments and merged
segment 2. added the total number of documents for input and merged segment
This is an automated email from the ASF dual-hosted git repository.
snlee pushed a commit to branch improve-merge-command
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit c6a25f27603743460fdf65113e2e9cd03ef682ef
Author: Seunghyun Lee <sn...@linkedin.com>
AuthorDate: Fri Nov 30 16:29:39 2018 -0800
Improve the logging for segment merge command
1. add min start / max end time for input segments and merged segment
2. added the total number of documents for input and merged segment
---
.../segment/converter/SegmentMergeCommand.java | 47 +++++++++++++---------
1 file changed, 27 insertions(+), 20 deletions(-)
diff --git a/pinot-tools/src/main/java/com/linkedin/pinot/tools/segment/converter/SegmentMergeCommand.java b/pinot-tools/src/main/java/com/linkedin/pinot/tools/segment/converter/SegmentMergeCommand.java
index 1c0da29..8cddcad 100644
--- a/pinot-tools/src/main/java/com/linkedin/pinot/tools/segment/converter/SegmentMergeCommand.java
+++ b/pinot-tools/src/main/java/com/linkedin/pinot/tools/segment/converter/SegmentMergeCommand.java
@@ -151,9 +151,27 @@ public class SegmentMergeCommand extends AbstractBaseAdminCommand implements Com
LOGGER.info("Table config: {}", tableConfig);
LOGGER.info("Schema : {}", schema);
+ // Compute mix/max time from segment metadata
+ long minStartTime = Long.MAX_VALUE;
+ long maxEndTime = Long.MIN_VALUE;
+ long totalNumDocsBeforeMerge = 0;
+ for (File indexDir : inputIndexDirs) {
+ SegmentMetadata segmentMetadata = new SegmentMetadataImpl(indexDir);
+ long currentStartTime = segmentMetadata.getStartTime();
+ if (currentStartTime < minStartTime) {
+ minStartTime = currentStartTime;
+ }
+
+ long currentEndTime = segmentMetadata.getEndTime();
+ if (currentEndTime > maxEndTime) {
+ maxEndTime = currentEndTime;
+ }
+ totalNumDocsBeforeMerge += segmentMetadata.getTotalDocs();
+ }
+
// Compute segment name if it is not specified
if (_outputSegmentName == null) {
- _outputSegmentName = getDefaultSegmentName(tableConfig, schema, inputIndexDirs);
+ _outputSegmentName = getDefaultSegmentName(tableConfig, schema, inputIndexDirs, minStartTime, maxEndTime);
}
LOGGER.info("Output segment name: {}", _outputSegmentName);
@@ -195,6 +213,11 @@ public class SegmentMergeCommand extends AbstractBaseAdminCommand implements Com
FileUtils.moveDirectory(outputSegment, finalOutputPath);
}
LOGGER.info("Segment has been merged correctly. Output file is located at {}", finalOutputPath);
+ LOGGER.info("Min start time / max end time for input segments : " + minStartTime + " / " + maxEndTime);
+ LOGGER.info("Min start time / max end time for merged segment: " + outputSegmentMetadata.getStartTime() + " / "
+ + outputSegmentMetadata.getEndTime());
+ LOGGER.info("Total number of documents for input segments: " + totalNumDocsBeforeMerge);
+ LOGGER.info("Total number of documents for merged segment: " + outputSegmentMetadata.getTotalDocs());
} finally {
// Clean up working directory
FileUtils.deleteQuietly(workingDir);
@@ -207,25 +230,9 @@ public class SegmentMergeCommand extends AbstractBaseAdminCommand implements Com
return "Create the merged segment using concatenation";
}
- private String getDefaultSegmentName(TableConfig tableConfig, Schema schema, List<File> inputIndexDirs)
- throws Exception {
- String tableName = TableNameBuilder.extractRawTableName(tableConfig.getTableName());
-
- // Compute mix/max time from segment metadata
- long minStartTime = Long.MAX_VALUE;
- long maxEndTime = Long.MIN_VALUE;
- for (File indexDir : inputIndexDirs) {
- SegmentMetadata segmentMetadata = new SegmentMetadataImpl(indexDir);
- long currentStartTime = segmentMetadata.getStartTime();
- if (currentStartTime < minStartTime) {
- minStartTime = currentStartTime;
- }
-
- long currentEndTime = segmentMetadata.getEndTime();
- if (currentEndTime > maxEndTime) {
- maxEndTime = currentEndTime;
- }
- }
+ private String getDefaultSegmentName(TableConfig tableConfig, Schema schema, List<File> inputIndexDirs,
+ long minStartTime, long maxEndTime) throws Exception {
+ String tableName = tableConfig.getTableName();
// Fetch time related configurations from schema and table config.
String pushFrequency = tableConfig.getValidationConfig().getSegmentPushFrequency();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org