You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by sn...@apache.org on 2018/12/01 00:30:19 UTC
[incubator-pinot] branch improve-merge-command updated (5a8ef50 ->
c6a25f2)
This is an automated email from the ASF dual-hosted git repository.
snlee pushed a change to branch improve-merge-command
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
discard 5a8ef50 Improve the logging for segment merge command 1. add min start / max end time for input segments and merged segment 2. added the total number of documents for input and merged segment
discard cc3b86a initial commit
new c6a25f2 Improve the logging for segment merge command 1. add min start / max end time for input segments and merged segment 2. added the total number of documents for input and merged segment
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (5a8ef50)
\
N -- N -- N refs/heads/improve-merge-command (c6a25f2)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../routing/HelixExternalViewBasedRouting.java | 22 +-
.../BasePartitionAwareRoutingTableBuilder.java | 6 +-
.../routing/builder/BaseRoutingTableBuilder.java | 11 +-
.../builder/DefaultOfflineRoutingTableBuilder.java | 5 +-
.../DefaultRealtimeRoutingTableBuilder.java | 11 +-
.../routing/builder/RoutingTableBuilder.java | 3 +-
.../routing/selector/DefaultSegmentSelector.java | 42 ---
.../routing/selector/MergedSegmentSelector.java | 92 ------
.../broker/routing/selector/SegmentSelector.java | 46 ---
.../routing/selector/SegmentSelectorFactory.java | 40 ---
.../{util => routing}/FakePropertyStore.java | 13 +-
.../broker/routing/RoutingTableBenchmark.java | 122 --------
.../pinot/broker/routing/RoutingTableTest.java | 1 -
.../BalancedRandomRoutingTableBuilderTest.java | 2 +-
.../HighLevelConsumerRoutingTableBuilderTest.java | 2 +-
...rtitionAwareOfflineRoutingTableBuilderTest.java | 9 +-
...titionAwareRealtimeRoutingTableBuilderTest.java | 11 +-
.../broker/selector/MergeSegmentSelectorTest.java | 77 -----
pinot-common/pom.xml | 4 -
.../common/config/MultiLevelRollupSetting.java | 77 -----
.../linkedin/pinot/common/config/RollupConfig.java | 59 ----
.../pinot/common/config/SegmentMergeConfig.java | 137 ---------
.../linkedin/pinot/common/config/TableConfig.java | 84 ++----
.../pinot/common/lineage/SegmentGroup.java | 72 -----
.../pinot/common/lineage/SegmentMergeLineage.java | 333 ---------------------
.../lineage/SegmentMergeLineageAccessHelper.java | 82 -----
.../pinot/common/metadata/ZKMetadataProvider.java | 5 -
.../common/utils/FileUploadDownloadClient.java | 51 +---
.../com/linkedin/pinot/common/utils/UUIDUtils.java | 29 --
.../common/lineage/SegmentMergeLineageTest.java | 106 -------
.../PinotSegmentUploadRestletResource.java | 9 +-
.../api/resources/PinotTableRestletResource.java | 48 ---
.../pinot/controller/api/upload/ZKOperator.java | 53 +---
.../helix/ControllerRequestURLBuilder.java | 5 -
.../helix/core/PinotHelixResourceManager.java | 68 +----
.../helix/core/minion/ClusterInfoProvider.java | 6 -
.../generator/SegmentMergeRollupTaskGenerator.java | 313 -------------------
.../minion/generator/TaskGeneratorRegistry.java | 1 -
.../pinot/core/common/MinionConstants.java | 8 -
.../data/readers/PinotSegmentRecordReader.java | 3 -
.../tests/SimpleMinionClusterIntegrationTest.java | 2 -
.../BaseMultipleSegmentsConversionExecutor.java | 13 +-
.../minion/executor/SegmentConversionUtils.java | 50 ----
.../executor/SegmentMergeRollupTaskExecutor.java | 105 -------
.../SegmentMergeRollupTaskExecutorFactory.java | 23 --
.../executor/TaskExecutorFactoryRegistry.java | 2 -
pom.xml | 5 -
47 files changed, 79 insertions(+), 2189 deletions(-)
delete mode 100644 pinot-broker/src/main/java/com/linkedin/pinot/broker/routing/selector/DefaultSegmentSelector.java
delete mode 100644 pinot-broker/src/main/java/com/linkedin/pinot/broker/routing/selector/MergedSegmentSelector.java
delete mode 100644 pinot-broker/src/main/java/com/linkedin/pinot/broker/routing/selector/SegmentSelector.java
delete mode 100644 pinot-broker/src/main/java/com/linkedin/pinot/broker/routing/selector/SegmentSelectorFactory.java
rename pinot-broker/src/test/java/com/linkedin/pinot/broker/{util => routing}/FakePropertyStore.java (88%)
delete mode 100644 pinot-broker/src/test/java/com/linkedin/pinot/broker/routing/RoutingTableBenchmark.java
delete mode 100644 pinot-broker/src/test/java/com/linkedin/pinot/broker/selector/MergeSegmentSelectorTest.java
delete mode 100644 pinot-common/src/main/java/com/linkedin/pinot/common/config/MultiLevelRollupSetting.java
delete mode 100644 pinot-common/src/main/java/com/linkedin/pinot/common/config/RollupConfig.java
delete mode 100644 pinot-common/src/main/java/com/linkedin/pinot/common/config/SegmentMergeConfig.java
delete mode 100644 pinot-common/src/main/java/com/linkedin/pinot/common/lineage/SegmentGroup.java
delete mode 100644 pinot-common/src/main/java/com/linkedin/pinot/common/lineage/SegmentMergeLineage.java
delete mode 100644 pinot-common/src/main/java/com/linkedin/pinot/common/lineage/SegmentMergeLineageAccessHelper.java
delete mode 100644 pinot-common/src/main/java/com/linkedin/pinot/common/utils/UUIDUtils.java
delete mode 100644 pinot-common/src/test/java/com/linkedin/pinot/common/lineage/SegmentMergeLineageTest.java
delete mode 100644 pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/minion/generator/SegmentMergeRollupTaskGenerator.java
delete mode 100644 pinot-minion/src/main/java/com/linkedin/pinot/minion/executor/SegmentMergeRollupTaskExecutor.java
delete mode 100644 pinot-minion/src/main/java/com/linkedin/pinot/minion/executor/SegmentMergeRollupTaskExecutorFactory.java
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[incubator-pinot] 01/01: Improve the logging for segment merge
command 1. add min start / max end time for input segments and merged
segment 2. added the total number of documents for input and merged segment
Posted by sn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
snlee pushed a commit to branch improve-merge-command
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit c6a25f27603743460fdf65113e2e9cd03ef682ef
Author: Seunghyun Lee <sn...@linkedin.com>
AuthorDate: Fri Nov 30 16:29:39 2018 -0800
Improve the logging for segment merge command
1. add min start / max end time for input segments and merged segment
2. added the total number of documents for input and merged segment
---
.../segment/converter/SegmentMergeCommand.java | 47 +++++++++++++---------
1 file changed, 27 insertions(+), 20 deletions(-)
diff --git a/pinot-tools/src/main/java/com/linkedin/pinot/tools/segment/converter/SegmentMergeCommand.java b/pinot-tools/src/main/java/com/linkedin/pinot/tools/segment/converter/SegmentMergeCommand.java
index 1c0da29..8cddcad 100644
--- a/pinot-tools/src/main/java/com/linkedin/pinot/tools/segment/converter/SegmentMergeCommand.java
+++ b/pinot-tools/src/main/java/com/linkedin/pinot/tools/segment/converter/SegmentMergeCommand.java
@@ -151,9 +151,27 @@ public class SegmentMergeCommand extends AbstractBaseAdminCommand implements Com
LOGGER.info("Table config: {}", tableConfig);
LOGGER.info("Schema : {}", schema);
+ // Compute mix/max time from segment metadata
+ long minStartTime = Long.MAX_VALUE;
+ long maxEndTime = Long.MIN_VALUE;
+ long totalNumDocsBeforeMerge = 0;
+ for (File indexDir : inputIndexDirs) {
+ SegmentMetadata segmentMetadata = new SegmentMetadataImpl(indexDir);
+ long currentStartTime = segmentMetadata.getStartTime();
+ if (currentStartTime < minStartTime) {
+ minStartTime = currentStartTime;
+ }
+
+ long currentEndTime = segmentMetadata.getEndTime();
+ if (currentEndTime > maxEndTime) {
+ maxEndTime = currentEndTime;
+ }
+ totalNumDocsBeforeMerge += segmentMetadata.getTotalDocs();
+ }
+
// Compute segment name if it is not specified
if (_outputSegmentName == null) {
- _outputSegmentName = getDefaultSegmentName(tableConfig, schema, inputIndexDirs);
+ _outputSegmentName = getDefaultSegmentName(tableConfig, schema, inputIndexDirs, minStartTime, maxEndTime);
}
LOGGER.info("Output segment name: {}", _outputSegmentName);
@@ -195,6 +213,11 @@ public class SegmentMergeCommand extends AbstractBaseAdminCommand implements Com
FileUtils.moveDirectory(outputSegment, finalOutputPath);
}
LOGGER.info("Segment has been merged correctly. Output file is located at {}", finalOutputPath);
+ LOGGER.info("Min start time / max end time for input segments : " + minStartTime + " / " + maxEndTime);
+ LOGGER.info("Min start time / max end time for merged segment: " + outputSegmentMetadata.getStartTime() + " / "
+ + outputSegmentMetadata.getEndTime());
+ LOGGER.info("Total number of documents for input segments: " + totalNumDocsBeforeMerge);
+ LOGGER.info("Total number of documents for merged segment: " + outputSegmentMetadata.getTotalDocs());
} finally {
// Clean up working directory
FileUtils.deleteQuietly(workingDir);
@@ -207,25 +230,9 @@ public class SegmentMergeCommand extends AbstractBaseAdminCommand implements Com
return "Create the merged segment using concatenation";
}
- private String getDefaultSegmentName(TableConfig tableConfig, Schema schema, List<File> inputIndexDirs)
- throws Exception {
- String tableName = TableNameBuilder.extractRawTableName(tableConfig.getTableName());
-
- // Compute mix/max time from segment metadata
- long minStartTime = Long.MAX_VALUE;
- long maxEndTime = Long.MIN_VALUE;
- for (File indexDir : inputIndexDirs) {
- SegmentMetadata segmentMetadata = new SegmentMetadataImpl(indexDir);
- long currentStartTime = segmentMetadata.getStartTime();
- if (currentStartTime < minStartTime) {
- minStartTime = currentStartTime;
- }
-
- long currentEndTime = segmentMetadata.getEndTime();
- if (currentEndTime > maxEndTime) {
- maxEndTime = currentEndTime;
- }
- }
+ private String getDefaultSegmentName(TableConfig tableConfig, Schema schema, List<File> inputIndexDirs,
+ long minStartTime, long maxEndTime) throws Exception {
+ String tableName = tableConfig.getTableName();
// Fetch time related configurations from schema and table config.
String pushFrequency = tableConfig.getValidationConfig().getSegmentPushFrequency();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org