You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by nf...@apache.org on 2023/12/18 15:57:27 UTC
(jackrabbit-oak) branch trunk updated: OAK-10571 - Names of metrics exported by indexing logic are inconsistent (#1241)
This is an automated email from the ASF dual-hosted git repository.
nfsantos pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push:
new c069c953f6 OAK-10571 - Names of metrics exported by indexing logic are inconsistent (#1241)
c069c953f6 is described below
commit c069c953f6738b4202202d9dad206c35395c1bf1
Author: Nuno Santos <ns...@adobe.com>
AuthorDate: Mon Dec 18 15:57:21 2023 +0000
OAK-10571 - Names of metrics exported by indexing logic are inconsistent (#1241)
* Rename metrics to follow a more consistent naming convention.
* Add missing metrics.
---
.../flatfile/pipelined/PipelinedMergeSortTask.java | 20 ++++++-----
.../flatfile/pipelined/PipelinedMetrics.java | 42 +++++++++++++---------
.../pipelined/PipelinedMongoDownloadTask.java | 9 ++++-
.../flatfile/pipelined/PipelinedSortBatchTask.java | 16 ++++++++-
.../flatfile/pipelined/PipelinedStrategy.java | 7 ++--
.../flatfile/pipelined/PipelinedUtils.java | 7 ++++
.../pipelined/TransformStageStatistics.java | 28 ++++++++-------
.../document/flatfile/pipelined/PipelinedIT.java | 39 +++++++++++---------
.../pipelined/PipelinedMergeSortTaskTest.java | 13 +++----
.../pipelined/PipelinedMongoDownloadTaskTest.java | 6 +++-
.../pipelined/PipelinedSortBatchTaskTest.java | 5 +--
11 files changed, 121 insertions(+), 71 deletions(-)
diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMergeSortTask.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMergeSortTask.java
index bf4083e796..c8b1a7fa5c 100644
--- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMergeSortTask.java
+++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMergeSortTask.java
@@ -49,10 +49,6 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Function;
import java.util.stream.Collectors;
-import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_EAGER_MERGES_RUNS;
-import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_FILES_COUNT;
-import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_TIME;
-import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_INTERMEDIATE_FILES_COUNT;
import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedStrategy.SENTINEL_SORTED_FILES_QUEUE;
import static org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreUtils.getSortedStoreFileName;
@@ -223,16 +219,24 @@ public class PipelinedMergeSortTask implements Callable<PipelinedMergeSortTask.R
String metrics = MetricsFormatter.newBuilder()
.add("duration", FormattingUtils.formatToSeconds(w))
.add("durationSeconds", durationSeconds)
+ .add("intermediateFilesCount", intermediateFilesCount)
+ .add("eagerMergesRuns", eagerMergeRuns)
.add("filesMerged", sortedFiles.size())
.add("ffsSizeBytes", ffsSizeBytes)
.add("ffsSize", IOUtils.humanReadableByteCountBin(ffsSizeBytes))
.build();
LOG.info("[TASK:{}:END] Metrics: {}", THREAD_NAME.toUpperCase(Locale.ROOT), metrics);
- MetricsUtils.setCounterOnce(statisticsProvider, OAK_INDEXER_PIPELINED_MERGE_SORT_INTERMEDIATE_FILES_COUNT, intermediateFilesCount);
- MetricsUtils.setCounterOnce(statisticsProvider, OAK_INDEXER_PIPELINED_MERGE_SORT_EAGER_MERGES_RUNS, eagerMergeRuns);
- MetricsUtils.setCounterOnce(statisticsProvider, OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_FILES_COUNT, sortedFiles.size());
- MetricsUtils.setCounterOnce(statisticsProvider, OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_TIME, durationSeconds);
+ MetricsUtils.setCounterOnce(statisticsProvider,
+ PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_DURATION_SECONDS, durationSeconds);
+ MetricsUtils.setCounterOnce(statisticsProvider,
+ PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_INTERMEDIATE_FILES_TOTAL, intermediateFilesCount);
+ MetricsUtils.setCounterOnce(statisticsProvider,
+ PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_EAGER_MERGES_RUNS_TOTAL, eagerMergeRuns);
+ MetricsUtils.setCounterOnce(statisticsProvider,
+ PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_FILES_COUNT_TOTAL, sortedFiles.size());
+ MetricsUtils.setCounterOnce(statisticsProvider,
+ PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FLAT_FILE_STORE_SIZE_BYTES, ffsSizeBytes);
return new Result(flatFileStore, intermediateFilesCount, sortedFiles.size(), eagerMergeRuns);
} else {
diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMetrics.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMetrics.java
index f44f79af40..f03cd75082 100644
--- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMetrics.java
+++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMetrics.java
@@ -19,24 +19,32 @@
package org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined;
public final class PipelinedMetrics {
+ // Conventions for naming metrics: https://prometheus.io/docs/practices/naming/
public static final String METRIC_NAME_PREFIX = "oak_indexer_pipelined_";
- public static final String OAK_INDEXER_PIPELINED_DOCUMENTS_DOWNLOADED = METRIC_NAME_PREFIX + "documentsDownloaded";
- public static final String OAK_INDEXER_PIPELINED_DOCUMENTS_TRAVERSED = METRIC_NAME_PREFIX + "documentsTraversed";
- public static final String OAK_INDEXER_PIPELINED_DOCUMENTS_REJECTED_SPLIT = METRIC_NAME_PREFIX + "documentsRejectedSplit";
- public static final String OAK_INDEXER_PIPELINED_DOCUMENTS_ACCEPTED = METRIC_NAME_PREFIX + "documentsAccepted";
- public static final String OAK_INDEXER_PIPELINED_DOCUMENTS_REJECTED = METRIC_NAME_PREFIX + "documentsRejected";
- public static final String OAK_INDEXER_PIPELINED_DOCUMENTS_REJECTED_EMPTY_NODE_STATE = METRIC_NAME_PREFIX + "documentsRejectedEmptyNodeState";
- public static final String OAK_INDEXER_PIPELINED_ENTRIES_TRAVERSED = METRIC_NAME_PREFIX + "entriesTraversed";
- public static final String OAK_INDEXER_PIPELINED_ENTRIES_ACCEPTED = METRIC_NAME_PREFIX + "entriesAccepted";
- public static final String OAK_INDEXER_PIPELINED_ENTRIES_REJECTED = METRIC_NAME_PREFIX + "entriesRejected";
- public static final String OAK_INDEXER_PIPELINED_ENTRIES_REJECTED_HIDDEN_PATHS = METRIC_NAME_PREFIX + "entriesRejectedHiddenPaths";
- public static final String OAK_INDEXER_PIPELINED_ENTRIES_REJECTED_PATH_FILTERED = METRIC_NAME_PREFIX + "entriesRejectedPathFiltered";
- public static final String OAK_INDEXER_PIPELINED_EXTRACTED_ENTRIES_TOTAL_SIZE = METRIC_NAME_PREFIX + "extractedEntriesTotalSize";
- public static final String OAK_INDEXER_PIPELINED_MONGO_DOWNLOAD_ENQUEUE_DELAY_PERCENTAGE = METRIC_NAME_PREFIX + "mongoDownloadEnqueueDelayPercentage";
- public static final String OAK_INDEXER_PIPELINED_MERGE_SORT_INTERMEDIATE_FILES_COUNT = METRIC_NAME_PREFIX + "mergeSortIntermediateFilesCount";
- public static final String OAK_INDEXER_PIPELINED_MERGE_SORT_EAGER_MERGES_RUNS = METRIC_NAME_PREFIX + "mergeSortEagerMergesRuns";
- public static final String OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_FILES_COUNT = METRIC_NAME_PREFIX + "mergeSortFinalMergeFilesCount";
- public static final String OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_TIME = METRIC_NAME_PREFIX + "mergeSortFinalMergeTime";
+ public static final String OAK_INDEXER_PIPELINED_MONGO_DOWNLOAD_DURATION_SECONDS = METRIC_NAME_PREFIX + "mongo_download_duration_seconds";
+ public static final String OAK_INDEXER_PIPELINED_MONGO_DOWNLOAD_ENQUEUE_DELAY_PERCENTAGE = METRIC_NAME_PREFIX + "mongo_download_enqueue_delay_percentage";
+ public static final String OAK_INDEXER_PIPELINED_DOCUMENTS_DOWNLOADED_TOTAL = METRIC_NAME_PREFIX + "documents_downloaded_total";
+ public static final String OAK_INDEXER_PIPELINED_DOCUMENTS_TRAVERSED_TOTAL = METRIC_NAME_PREFIX + "documents_traversed_total";
+ public static final String OAK_INDEXER_PIPELINED_DOCUMENTS_REJECTED_SPLIT_TOTAL = METRIC_NAME_PREFIX + "documents_rejected_split_total";
+ public static final String OAK_INDEXER_PIPELINED_DOCUMENTS_ACCEPTED_TOTAL = METRIC_NAME_PREFIX + "documents_accepted_total";
+ public static final String OAK_INDEXER_PIPELINED_DOCUMENTS_REJECTED_TOTAL = METRIC_NAME_PREFIX + "documents_rejected_total";
+ public static final String OAK_INDEXER_PIPELINED_DOCUMENTS_ACCEPTED_PERCENTAGE = METRIC_NAME_PREFIX + "documents_accepted_percentage";
+ public static final String OAK_INDEXER_PIPELINED_DOCUMENTS_REJECTED_EMPTY_NODE_STATE_TOTAL = METRIC_NAME_PREFIX + "documents_rejected_empty_node_state_total";
+ public static final String OAK_INDEXER_PIPELINED_ENTRIES_TRAVERSED_TOTAL = METRIC_NAME_PREFIX + "entries_traversed_total";
+ public static final String OAK_INDEXER_PIPELINED_ENTRIES_ACCEPTED_TOTAL = METRIC_NAME_PREFIX + "entries_accepted_total";
+ public static final String OAK_INDEXER_PIPELINED_ENTRIES_ACCEPTED_PERCENTAGE = METRIC_NAME_PREFIX + "entries_accepted_percentage";
+ public static final String OAK_INDEXER_PIPELINED_ENTRIES_REJECTED_TOTAL = METRIC_NAME_PREFIX + "entries_rejected_total";
+ public static final String OAK_INDEXER_PIPELINED_ENTRIES_REJECTED_HIDDEN_PATHS_TOTAL = METRIC_NAME_PREFIX + "entries_rejected_hidden_paths_total";
+ public static final String OAK_INDEXER_PIPELINED_ENTRIES_REJECTED_PATH_FILTERED_TOTAL = METRIC_NAME_PREFIX + "entries_rejected_path_filtered_total";
+ public static final String OAK_INDEXER_PIPELINED_EXTRACTED_ENTRIES_TOTAL_BYTES = METRIC_NAME_PREFIX + "extracted_entries_total_bytes";
+ public static final String OAK_INDEXER_PIPELINED_SORT_BATCH_PHASE_CREATE_SORT_ARRAY_PERCENTAGE = METRIC_NAME_PREFIX + "sort_batch_phase_create_sort_array_percentage";
+ public static final String OAK_INDEXER_PIPELINED_SORT_BATCH_PHASE_SORT_ARRAY_PERCENTAGE = METRIC_NAME_PREFIX + "sort_batch_phase_sort_array_percentage";
+ public static final String OAK_INDEXER_PIPELINED_SORT_BATCH_PHASE_WRITE_TO_DISK_PERCENTAGE = METRIC_NAME_PREFIX + "sort_batch_phase_write_to_disk_percentage";
+ public static final String OAK_INDEXER_PIPELINED_MERGE_SORT_INTERMEDIATE_FILES_TOTAL = METRIC_NAME_PREFIX + "merge_sort_intermediate_files_total";
+ public static final String OAK_INDEXER_PIPELINED_MERGE_SORT_EAGER_MERGES_RUNS_TOTAL = METRIC_NAME_PREFIX + "merge_sort_eager_merges_runs_total";
+ public static final String OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_FILES_COUNT_TOTAL = METRIC_NAME_PREFIX + "merge_sort_final_merge_files_total";
+ public static final String OAK_INDEXER_PIPELINED_MERGE_SORT_FLAT_FILE_STORE_SIZE_BYTES = METRIC_NAME_PREFIX + "merge_sort_flat_file_store_size_bytes";
+ public static final String OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_DURATION_SECONDS = METRIC_NAME_PREFIX + "merge_sort_final_merge_duration_seconds";
private PipelinedMetrics() {
}
diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMongoDownloadTask.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMongoDownloadTask.java
index caa8a1ee75..00c5c8f792 100644
--- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMongoDownloadTask.java
+++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMongoDownloadTask.java
@@ -206,7 +206,14 @@ public class PipelinedMongoDownloadTask implements Callable<PipelinedMongoDownlo
.add("enqueueingDelayMillis", totalEnqueueWaitTimeMillis)
.add("enqueueingDelayPercentage", enqueueingDelayPercentage)
.build();
-
+ MetricsUtils.setCounterOnce(statisticsProvider,
+ PipelinedMetrics.OAK_INDEXER_PIPELINED_MONGO_DOWNLOAD_DURATION_SECONDS,
+ durationMillis / 1000
+ );
+ MetricsUtils.setCounterOnce(statisticsProvider,
+ PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_DOWNLOADED_TOTAL,
+ documentsRead
+ );
MetricsUtils.setCounterOnce(statisticsProvider,
PipelinedMetrics.OAK_INDEXER_PIPELINED_MONGO_DOWNLOAD_ENQUEUE_DELAY_PERCENTAGE,
PipelinedUtils.toPercentage(totalEnqueueWaitTimeMillis, durationMillis)
diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedSortBatchTask.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedSortBatchTask.java
index c53c72a955..26f85e73a1 100644
--- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedSortBatchTask.java
+++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedSortBatchTask.java
@@ -23,6 +23,8 @@ import org.apache.jackrabbit.guava.common.base.Stopwatch;
import org.apache.jackrabbit.oak.commons.Compression;
import org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreUtils;
import org.apache.jackrabbit.oak.plugins.index.MetricsFormatter;
+import org.apache.jackrabbit.oak.plugins.index.MetricsUtils;
+import org.apache.jackrabbit.oak.stats.StatisticsProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -68,6 +70,7 @@ class PipelinedSortBatchTask implements Callable<PipelinedSortBatchTask.Result>
private final BlockingQueue<NodeStateEntryBatch> nonEmptyBuffersQueue;
private final BlockingQueue<Path> sortedFilesQueue;
private final Path sortWorkDir;
+ private final StatisticsProvider statisticsProvider;
private final ArrayList<SortKey> sortBuffer = new ArrayList<>(32 * 1024);
private long entriesProcessed = 0;
private long batchesProcessed = 0;
@@ -80,13 +83,15 @@ class PipelinedSortBatchTask implements Callable<PipelinedSortBatchTask.Result>
Compression algorithm,
BlockingQueue<NodeStateEntryBatch> emptyBuffersQueue,
BlockingQueue<NodeStateEntryBatch> nonEmptyBuffersQueue,
- BlockingQueue<Path> sortedFilesQueue) throws IOException {
+ BlockingQueue<Path> sortedFilesQueue,
+ StatisticsProvider statisticsProvider) throws IOException {
this.pathComparator = (e1, e2) -> pathComparator.compare(e1.getPathElements(), e2.getPathElements());
this.algorithm = algorithm;
this.emptyBuffersQueue = emptyBuffersQueue;
this.nonEmptyBuffersQueue = nonEmptyBuffersQueue;
this.sortedFilesQueue = sortedFilesQueue;
this.sortWorkDir = createdSortWorkDir(storeDir);
+ this.statisticsProvider = statisticsProvider;
}
@Override
@@ -118,6 +123,15 @@ class PipelinedSortBatchTask implements Callable<PipelinedSortBatchTask.Result>
.add("totalTimeSeconds", totalTimeMillis / 1000)
.build();
LOG.info("[TASK:{}:END] Metrics: {}", THREAD_NAME.toUpperCase(Locale.ROOT), metrics);
+ MetricsUtils.setCounterOnce(statisticsProvider,
+ PipelinedMetrics.OAK_INDEXER_PIPELINED_SORT_BATCH_PHASE_CREATE_SORT_ARRAY_PERCENTAGE,
+ PipelinedUtils.toPercentage(timeCreatingSortArrayMillis, totalTimeMillis));
+ MetricsUtils.setCounterOnce(statisticsProvider,
+ PipelinedMetrics.OAK_INDEXER_PIPELINED_SORT_BATCH_PHASE_SORT_ARRAY_PERCENTAGE,
+ PipelinedUtils.toPercentage(timeSortingMillis, totalTimeMillis));
+ MetricsUtils.setCounterOnce(statisticsProvider,
+ PipelinedMetrics.OAK_INDEXER_PIPELINED_SORT_BATCH_PHASE_WRITE_TO_DISK_PERCENTAGE,
+ PipelinedUtils.toPercentage(timeWritingMillis, totalTimeMillis));
return new Result(entriesProcessed);
}
sortAndSaveBatch(nseBuffer);
diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedStrategy.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedStrategy.java
index 14bf4e8bdb..74fcce1f6a 100644
--- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedStrategy.java
+++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedStrategy.java
@@ -33,7 +33,6 @@ import org.apache.jackrabbit.oak.plugins.document.RevisionVector;
import org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentStore;
import org.apache.jackrabbit.oak.plugins.index.FormattingUtils;
import org.apache.jackrabbit.oak.plugins.index.MetricsFormatter;
-import org.apache.jackrabbit.oak.plugins.index.MetricsUtils;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.apache.jackrabbit.oak.spi.filter.PathFilter;
import org.apache.jackrabbit.oak.stats.StatisticsProvider;
@@ -62,7 +61,6 @@ import java.util.concurrent.TimeUnit;
import java.util.function.Predicate;
import static org.apache.jackrabbit.oak.commons.IOUtils.humanReadableByteCountBin;
-import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_DOWNLOADED;
/**
* Downloads the contents of the MongoDB repository dividing the tasks in a pipeline with the following stages:
@@ -435,7 +433,8 @@ public class PipelinedStrategy extends IndexStoreSortStrategyBase {
this.getAlgorithm(),
emptyBatchesQueue,
nonEmptyBatchesQueue,
- sortedFilesQueue
+ sortedFilesQueue,
+ statisticsProvider
));
PipelinedMergeSortTask mergeSortTask = new PipelinedMergeSortTask(
@@ -463,7 +462,6 @@ public class PipelinedStrategy extends IndexStoreSortStrategyBase {
mongoDocQueue.put(SENTINEL_MONGO_DOCUMENT);
}
mergeSortTask.stopEagerMerging();
- MetricsUtils.setCounterOnce(statisticsProvider, OAK_INDEXER_PIPELINED_DOCUMENTS_DOWNLOADED, downloadResult.getDocumentsDownloaded());
} else if (result instanceof PipelinedTransformTask.Result) {
PipelinedTransformTask.Result transformResult = (PipelinedTransformTask.Result) result;
@@ -527,7 +525,6 @@ public class PipelinedStrategy extends IndexStoreSortStrategyBase {
.add("durationSeconds", start.elapsed(TimeUnit.SECONDS))
.add("nodeStateEntriesExtracted", nodeStateEntriesExtracted)
.build());
- printStatistics(mongoDocQueue, emptyBatchesQueue, nonEmptyBatchesQueue, sortedFilesQueue, transformStageStatistics, true);
} catch (InterruptedException e) {
throw new RuntimeException(e);
} finally {
diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedUtils.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedUtils.java
index 85fad87ae0..50062d25b4 100644
--- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedUtils.java
+++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedUtils.java
@@ -55,4 +55,11 @@ public class PipelinedUtils {
}
}
+ public static int toPercentageAsInt(long numerator, long denominator) {
+ if (denominator == 0) {
+ return -1;
+ } else {
+ return (int) Math.round((100.0 * numerator) / denominator);
+ }
+ }
}
diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/TransformStageStatistics.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/TransformStageStatistics.java
index 61cf67e02a..2b70a8cc4a 100644
--- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/TransformStageStatistics.java
+++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/TransformStageStatistics.java
@@ -145,18 +145,22 @@ public class TransformStageStatistics {
long entriesAcceptedTotalSizeSum = entriesAcceptedTotalSize.sum();
long entriesRejectedSum = entriesRejected.sum();
long entriesTraversed = entriesAcceptedSum + entriesRejectedSum;
-
- MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_TRAVERSED, mongoDocumentsTraversedSum);
- MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_REJECTED_SPLIT, documentsRejectedSplitSum);
- MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_REJECTED_EMPTY_NODE_STATE, documentsRejectedEmptyNodeStateSum);
- MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_ACCEPTED, documentsAcceptedTotal);
- MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_REJECTED, documentsRejectedTotal);
- MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_ENTRIES_TRAVERSED, entriesTraversed);
- MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_ENTRIES_ACCEPTED, entriesAcceptedSum);
- MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_ENTRIES_REJECTED, entriesRejectedSum);
- MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_ENTRIES_REJECTED_HIDDEN_PATHS, entriesRejectedHiddenPaths.sum());
- MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_ENTRIES_REJECTED_PATH_FILTERED, entriesRejectedPathFiltered.sum());
- MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_EXTRACTED_ENTRIES_TOTAL_SIZE, entriesAcceptedTotalSizeSum);
+ int documentsAcceptedPercentage = PipelinedUtils.toPercentageAsInt(documentsAcceptedTotal, mongoDocumentsTraversedSum);
+ int entriesAcceptedPercentage = PipelinedUtils.toPercentageAsInt(entriesAcceptedSum, entriesAcceptedSum + entriesRejectedSum);
+
+ MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_TRAVERSED_TOTAL, mongoDocumentsTraversedSum);
+ MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_REJECTED_SPLIT_TOTAL, documentsRejectedSplitSum);
+ MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_REJECTED_EMPTY_NODE_STATE_TOTAL, documentsRejectedEmptyNodeStateSum);
+ MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_ACCEPTED_TOTAL, documentsAcceptedTotal);
+ MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_REJECTED_TOTAL, documentsRejectedTotal);
+ MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_ACCEPTED_PERCENTAGE, documentsAcceptedPercentage);
+ MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_ENTRIES_TRAVERSED_TOTAL, entriesTraversed);
+ MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_ENTRIES_ACCEPTED_TOTAL, entriesAcceptedSum);
+ MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_ENTRIES_ACCEPTED_PERCENTAGE, entriesAcceptedPercentage);
+ MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_ENTRIES_REJECTED_TOTAL, entriesRejectedSum);
+ MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_ENTRIES_REJECTED_HIDDEN_PATHS_TOTAL, entriesRejectedHiddenPaths.sum());
+ MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_ENTRIES_REJECTED_PATH_FILTERED_TOTAL, entriesRejectedPathFiltered.sum());
+ MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_EXTRACTED_ENTRIES_TOTAL_BYTES, entriesAcceptedTotalSizeSum);
}
public String formatStats() {
diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedIT.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedIT.java
index 1da9b7f86e..122342f5d2 100644
--- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedIT.java
+++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedIT.java
@@ -249,23 +249,30 @@ public class PipelinedIT {
Set<String> metricsNames = statsProvider.getRegistry().getCounters().keySet();
assertEquals(Set.of(
- PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_DOWNLOADED,
- PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_TRAVERSED,
- PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_REJECTED_SPLIT,
- PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_ACCEPTED,
- PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_REJECTED,
- PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_REJECTED_EMPTY_NODE_STATE,
- PipelinedMetrics.OAK_INDEXER_PIPELINED_ENTRIES_TRAVERSED,
- PipelinedMetrics.OAK_INDEXER_PIPELINED_ENTRIES_ACCEPTED,
- PipelinedMetrics.OAK_INDEXER_PIPELINED_ENTRIES_REJECTED,
- PipelinedMetrics.OAK_INDEXER_PIPELINED_ENTRIES_REJECTED_HIDDEN_PATHS,
- PipelinedMetrics.OAK_INDEXER_PIPELINED_ENTRIES_REJECTED_PATH_FILTERED,
- PipelinedMetrics.OAK_INDEXER_PIPELINED_EXTRACTED_ENTRIES_TOTAL_SIZE,
+ PipelinedMetrics.OAK_INDEXER_PIPELINED_MONGO_DOWNLOAD_DURATION_SECONDS,
PipelinedMetrics.OAK_INDEXER_PIPELINED_MONGO_DOWNLOAD_ENQUEUE_DELAY_PERCENTAGE,
- PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_INTERMEDIATE_FILES_COUNT,
- PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_EAGER_MERGES_RUNS,
- PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_FILES_COUNT,
- PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_TIME
+ PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_DOWNLOADED_TOTAL,
+ PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_TRAVERSED_TOTAL,
+ PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_REJECTED_SPLIT_TOTAL,
+ PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_ACCEPTED_TOTAL,
+ PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_REJECTED_TOTAL,
+ PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_ACCEPTED_PERCENTAGE,
+ PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_REJECTED_EMPTY_NODE_STATE_TOTAL,
+ PipelinedMetrics.OAK_INDEXER_PIPELINED_ENTRIES_TRAVERSED_TOTAL,
+ PipelinedMetrics.OAK_INDEXER_PIPELINED_ENTRIES_ACCEPTED_TOTAL,
+ PipelinedMetrics.OAK_INDEXER_PIPELINED_ENTRIES_REJECTED_TOTAL,
+ PipelinedMetrics.OAK_INDEXER_PIPELINED_ENTRIES_ACCEPTED_PERCENTAGE,
+ PipelinedMetrics.OAK_INDEXER_PIPELINED_ENTRIES_REJECTED_HIDDEN_PATHS_TOTAL,
+ PipelinedMetrics.OAK_INDEXER_PIPELINED_ENTRIES_REJECTED_PATH_FILTERED_TOTAL,
+ PipelinedMetrics.OAK_INDEXER_PIPELINED_EXTRACTED_ENTRIES_TOTAL_BYTES,
+ PipelinedMetrics.OAK_INDEXER_PIPELINED_SORT_BATCH_PHASE_CREATE_SORT_ARRAY_PERCENTAGE,
+ PipelinedMetrics.OAK_INDEXER_PIPELINED_SORT_BATCH_PHASE_SORT_ARRAY_PERCENTAGE,
+ PipelinedMetrics.OAK_INDEXER_PIPELINED_SORT_BATCH_PHASE_WRITE_TO_DISK_PERCENTAGE,
+ PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_INTERMEDIATE_FILES_TOTAL,
+ PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_EAGER_MERGES_RUNS_TOTAL,
+ PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_FILES_COUNT_TOTAL,
+ PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FLAT_FILE_STORE_SIZE_BYTES,
+ PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_DURATION_SECONDS
), metricsNames);
String pipelinedMetrics = statsProvider.getRegistry()
diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMergeSortTaskTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMergeSortTaskTest.java
index 1dbc10632f..fc25bd7a8e 100644
--- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMergeSortTaskTest.java
+++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMergeSortTaskTest.java
@@ -39,10 +39,6 @@ import java.util.concurrent.ScheduledExecutorService;
import java.util.stream.Collectors;
import java.util.stream.Stream;
-import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_EAGER_MERGES_RUNS;
-import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_FILES_COUNT;
-import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_TIME;
-import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_INTERMEDIATE_FILES_COUNT;
import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedStrategy.FLATFILESTORE_CHARSET;
import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedStrategy.SENTINEL_SORTED_FILES_QUEUE;
import static org.junit.Assert.assertArrayEquals;
@@ -137,10 +133,11 @@ public class PipelinedMergeSortTaskTest extends PipelinedMergeSortTaskTestBase {
assertTrue(Files.exists(result.getFlatFileStoreFile()));
Set<String> metricNames = metricStatisticsProvider.getRegistry().getCounters().keySet();
assertEquals(metricNames, Set.of(
- OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_TIME,
- OAK_INDEXER_PIPELINED_MERGE_SORT_INTERMEDIATE_FILES_COUNT,
- OAK_INDEXER_PIPELINED_MERGE_SORT_EAGER_MERGES_RUNS,
- OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_FILES_COUNT
+ PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_DURATION_SECONDS,
+ PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_INTERMEDIATE_FILES_TOTAL,
+ PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_EAGER_MERGES_RUNS_TOTAL,
+ PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_FILES_COUNT_TOTAL,
+ PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FLAT_FILE_STORE_SIZE_BYTES
));
return result;
}
diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMongoDownloadTaskTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMongoDownloadTaskTest.java
index fe1ca75945..ae8f2ec5db 100644
--- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMongoDownloadTaskTest.java
+++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMongoDownloadTaskTest.java
@@ -121,7 +121,11 @@ public class PipelinedMongoDownloadTaskTest {
assertEquals(documents, actualDocuments);
Set<String> metricNames = metricStatisticsProvider.getRegistry().getCounters().keySet();
- assertEquals(metricNames, Set.of(PipelinedMetrics.OAK_INDEXER_PIPELINED_MONGO_DOWNLOAD_ENQUEUE_DELAY_PERCENTAGE));
+ assertEquals(metricNames, Set.of(
+ PipelinedMetrics.OAK_INDEXER_PIPELINED_MONGO_DOWNLOAD_ENQUEUE_DELAY_PERCENTAGE,
+ PipelinedMetrics.OAK_INDEXER_PIPELINED_MONGO_DOWNLOAD_DURATION_SECONDS,
+ PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_DOWNLOADED_TOTAL
+ ));
}
} finally {
executor.shutdown();
diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedSortBatchTaskTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedSortBatchTaskTest.java
index 2e88536fd0..735649b8b3 100644
--- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedSortBatchTaskTest.java
+++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedSortBatchTaskTest.java
@@ -19,6 +19,7 @@
package org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined;
import org.apache.jackrabbit.oak.commons.Compression;
+import org.apache.jackrabbit.oak.stats.StatisticsProvider;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -169,8 +170,8 @@ public class PipelinedSortBatchTaskTest {
nonEmptyBatchesQueue.put(PipelinedStrategy.SENTINEL_NSE_BUFFER);
PipelinedSortBatchTask sortTask = new PipelinedSortBatchTask(
- sortRoot, pathComparator, algorithm, emptyBatchesQueue, nonEmptyBatchesQueue, sortedFilesQueue
- );
+ sortRoot, pathComparator, algorithm, emptyBatchesQueue, nonEmptyBatchesQueue, sortedFilesQueue,
+ StatisticsProvider.NOOP);
PipelinedSortBatchTask.Result taskResult = sortTask.call();
LOG.info("Result: {}", taskResult.getTotalEntries());
LOG.info("Empty batches: {}", emptyBatchesQueue.size());