You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by yi...@apache.org on 2024/06/23 23:10:21 UTC
(hudi) branch master updated: [HUDI-7395] Fix computation for metrics in HoodieMetadataMetrics (#10641)
This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new dbc6ac50aec [HUDI-7395] Fix computation for metrics in HoodieMetadataMetrics (#10641)
dbc6ac50aec is described below
commit dbc6ac50aec2f9724c46d67c0ab4a3f9b2489b75
Author: Lokesh Jain <lj...@apache.org>
AuthorDate: Mon Jun 24 04:40:16 2024 +0530
[HUDI-7395] Fix computation for metrics in HoodieMetadataMetrics (#10641)
---
.../apache/hudi/metadata/HoodieBackedTableMetadataWriter.java | 8 ++++----
.../java/org/apache/hudi/metadata/HoodieMetadataWriteUtils.java | 5 +++++
.../main/java/org/apache/hudi/metadata/BaseTableMetadata.java | 9 +++++----
.../java/org/apache/hudi/metadata/HoodieMetadataMetrics.java | 9 ++++++---
4 files changed, 20 insertions(+), 11 deletions(-)
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
index 8dc738b59ed..89d21e79b22 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
@@ -299,7 +299,7 @@ public abstract class HoodieBackedTableMetadataWriter<I> implements HoodieTableM
}
if (reInitialize) {
- metrics.ifPresent(m -> m.updateMetrics(HoodieMetadataMetrics.REBOOTSTRAP_STR, 1));
+ metrics.ifPresent(m -> m.incrementMetric(HoodieMetadataMetrics.REBOOTSTRAP_STR, 1));
LOG.info("Deleting Metadata Table directory so that it can be re-initialized");
HoodieTableMetadataUtil.deleteMetadataTable(dataMetaClient, engineContext, false);
exists = false;
@@ -1484,11 +1484,11 @@ public abstract class HoodieBackedTableMetadataWriter<I> implements HoodieTableM
throw e;
} finally {
long timeSpent = metadataTableServicesTimer.endTimer();
- metrics.ifPresent(m -> m.updateMetrics(HoodieMetadataMetrics.TABLE_SERVICE_EXECUTION_DURATION, timeSpent));
+ metrics.ifPresent(m -> m.setMetric(HoodieMetadataMetrics.TABLE_SERVICE_EXECUTION_DURATION, timeSpent));
if (allTableServicesExecutedSuccessfullyOrSkipped) {
- metrics.ifPresent(m -> m.incrementMetric(HoodieMetadataMetrics.TABLE_SERVICE_EXECUTION_STATUS, 1));
+ metrics.ifPresent(m -> m.setMetric(HoodieMetadataMetrics.TABLE_SERVICE_EXECUTION_STATUS, 1));
} else {
- metrics.ifPresent(m -> m.incrementMetric(HoodieMetadataMetrics.TABLE_SERVICE_EXECUTION_STATUS, -1));
+ metrics.ifPresent(m -> m.setMetric(HoodieMetadataMetrics.TABLE_SERVICE_EXECUTION_STATUS, -1));
}
}
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataWriteUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataWriteUtils.java
index bd4c82a2d84..3caff83eb05 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataWriteUtils.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataWriteUtils.java
@@ -48,6 +48,7 @@ import java.util.Properties;
import static org.apache.hudi.common.config.HoodieMetadataConfig.DEFAULT_METADATA_ASYNC_CLEAN;
import static org.apache.hudi.common.config.HoodieMetadataConfig.DEFAULT_METADATA_CLEANER_COMMITS_RETAINED;
import static org.apache.hudi.common.config.HoodieMetadataConfig.DEFAULT_METADATA_POPULATE_META_FIELDS;
+import static org.apache.hudi.common.util.StringUtils.nonEmpty;
import static org.apache.hudi.metadata.HoodieTableMetadata.METADATA_TABLE_NAME_SUFFIX;
/**
@@ -160,6 +161,10 @@ public class HoodieMetadataWriteUtils {
final Properties properties = new Properties();
properties.put(HoodieTableConfig.RECORDKEY_FIELDS.key(), RECORD_KEY_FIELD_NAME);
properties.put("hoodie.datasource.write.recordkey.field", RECORD_KEY_FIELD_NAME);
+ if (nonEmpty(writeConfig.getMetricReporterMetricsNamePrefix())) {
+ properties.put(HoodieMetricsConfig.METRICS_REPORTER_PREFIX.key(),
+ writeConfig.getMetricReporterMetricsNamePrefix() + METADATA_TABLE_NAME_SUFFIX);
+ }
builder.withProperties(properties);
if (writeConfig.isMetricsOn()) {
diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java b/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java
index 633ab7cd1da..bc858a2a15e 100644
--- a/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java
+++ b/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java
@@ -207,8 +207,8 @@ public abstract class BaseTableMetadata extends AbstractHoodieTableMetadata {
List<String> partitionIDFileIDStringsList = new ArrayList<>(partitionIDFileIDStrings);
Map<String, HoodieRecord<HoodieMetadataPayload>> hoodieRecords =
getRecordsByKeys(partitionIDFileIDStringsList, MetadataPartitionType.BLOOM_FILTERS.getPartitionPath());
- metrics.ifPresent(m -> m.updateMetrics(HoodieMetadataMetrics.LOOKUP_BLOOM_FILTERS_METADATA_STR,
- (timer.endTimer() / partitionIDFileIDStrings.size())));
+ metrics.ifPresent(m -> m.updateMetrics(HoodieMetadataMetrics.LOOKUP_BLOOM_FILTERS_METADATA_STR, timer.endTimer()));
+ metrics.ifPresent(m -> m.setMetric(HoodieMetadataMetrics.LOOKUP_BLOOM_FILTERS_FILE_COUNT_STR, partitionIDFileIDStringsList.size()));
Map<Pair<String, String>, BloomFilter> partitionFileToBloomFilterMap = new HashMap<>(hoodieRecords.size());
for (final Map.Entry<String, HoodieRecord<HoodieMetadataPayload>> entry : hoodieRecords.entrySet()) {
@@ -259,6 +259,7 @@ public abstract class BaseTableMetadata extends AbstractHoodieTableMetadata {
Map<String, HoodieRecord<HoodieMetadataPayload>> hoodieRecords =
getRecordsByKeys(columnStatKeylist, MetadataPartitionType.COLUMN_STATS.getPartitionPath());
metrics.ifPresent(m -> m.updateMetrics(HoodieMetadataMetrics.LOOKUP_COLUMN_STATS_METADATA_STR, timer.endTimer()));
+ metrics.ifPresent(m -> m.setMetric(HoodieMetadataMetrics.LOOKUP_COLUMN_STATS_FILE_COUNT_STR, columnStatKeylist.size()));
Map<Pair<String, String>, HoodieMetadataColumnStats> fileToColumnStatMap = new HashMap<>();
for (final Map.Entry<String, HoodieRecord<HoodieMetadataPayload>> entry : hoodieRecords.entrySet()) {
@@ -306,8 +307,8 @@ public abstract class BaseTableMetadata extends AbstractHoodieTableMetadata {
}));
metrics.ifPresent(m -> m.updateMetrics(HoodieMetadataMetrics.LOOKUP_RECORD_INDEX_TIME_STR, timer.endTimer()));
- metrics.ifPresent(m -> m.updateMetrics(HoodieMetadataMetrics.LOOKUP_RECORD_INDEX_KEYS_COUNT_STR, recordKeys.size()));
- metrics.ifPresent(m -> m.updateMetrics(HoodieMetadataMetrics.LOOKUP_RECORD_INDEX_KEYS_HITS_COUNT_STR, recordKeyToLocation.size()));
+ metrics.ifPresent(m -> m.setMetric(HoodieMetadataMetrics.LOOKUP_RECORD_INDEX_KEYS_COUNT_STR, recordKeys.size()));
+ metrics.ifPresent(m -> m.setMetric(HoodieMetadataMetrics.LOOKUP_RECORD_INDEX_KEYS_HITS_COUNT_STR, recordKeyToLocation.size()));
return recordKeyToLocation;
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMetrics.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMetrics.java
index 25a79cb12d8..e7392a8f1ac 100644
--- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMetrics.java
+++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMetrics.java
@@ -52,16 +52,19 @@ public class HoodieMetadataMetrics implements Serializable {
public static final String LOOKUP_FILES_STR = "lookup_files";
public static final String LOOKUP_BLOOM_FILTERS_METADATA_STR = "lookup_meta_index_bloom_filters";
public static final String LOOKUP_COLUMN_STATS_METADATA_STR = "lookup_meta_index_column_ranges";
+ public static final String LOOKUP_BLOOM_FILTERS_FILE_COUNT_STR = "lookup_meta_index_bloom_filters_file_count";
+ public static final String LOOKUP_COLUMN_STATS_FILE_COUNT_STR = "lookup_meta_index_column_ranges_file_count";
+
// Time for lookup from record index
public static final String LOOKUP_RECORD_INDEX_TIME_STR = "lookup_record_index_time";
// Number of keys looked up in a call
public static final String LOOKUP_RECORD_INDEX_KEYS_COUNT_STR = "lookup_record_index_key_count";
// Number of keys found in record index
- public static final String LOOKUP_RECORD_INDEX_KEYS_HITS_COUNT_STR = "lookup_record_index_key_count";
+ public static final String LOOKUP_RECORD_INDEX_KEYS_HITS_COUNT_STR = "lookup_record_index_key_hit_count";
public static final String SCAN_STR = "scan";
public static final String BASEFILE_READ_STR = "basefile_read";
public static final String INITIALIZE_STR = "initialize";
- public static final String REBOOTSTRAP_STR = "rebootstrap";
+ public static final String REBOOTSTRAP_STR = "rebootstrap_count";
public static final String BOOTSTRAP_ERR_STR = "bootstrap_error";
// Stats names
@@ -143,7 +146,7 @@ public class HoodieMetadataMetrics implements Serializable {
String countKey = action + ".count";
String durationKey = action + ".totalDuration";
incrementMetric(countKey, 1);
- incrementMetric(durationKey, durationInMs);
+ setMetric(durationKey, durationInMs);
}
public void updateSizeMetrics(HoodieTableMetaClient metaClient, HoodieBackedTableMetadata metadata, Set<String> metadataPartitions) {