You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by sa...@apache.org on 2023/02/28 11:36:37 UTC
[ozone] branch master updated: HDDS-7183. Expose RocksDB critical properties. (#3875)
This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 25ad1f2a4f HDDS-7183. Expose RocksDB critical properties. (#3875)
25ad1f2a4f is described below
commit 25ad1f2a4fc944cff098c22edaea1bd9e08344ba
Author: Sammi Chen <sa...@apache.org>
AuthorDate: Tue Feb 28 19:36:31 2023 +0800
HDDS-7183. Expose RocksDB critical properties. (#3875)
---
.../hadoop/hdds/utils/RocksDBStoreMBean.java | 160 ++++++++++++++++++++-
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 3 +-
2 files changed, 157 insertions(+), 6 deletions(-)
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/RocksDBStoreMBean.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/RocksDBStoreMBean.java
index ba9a144a56..eed2b75592 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/RocksDBStoreMBean.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/RocksDBStoreMBean.java
@@ -18,14 +18,17 @@
package org.apache.hadoop.hdds.utils;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase;
import org.apache.hadoop.metrics2.MetricsCollector;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.metrics2.MetricsSource;
import org.apache.hadoop.metrics2.MetricsSystem;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.lib.Interns;
+import org.bouncycastle.util.Strings;
import org.rocksdb.HistogramData;
import org.rocksdb.HistogramType;
+import org.rocksdb.LiveFileMetaData;
import org.rocksdb.Statistics;
import org.rocksdb.TickerType;
import org.slf4j.Logger;
@@ -40,10 +43,13 @@ import javax.management.MBeanAttributeInfo;
import javax.management.MBeanException;
import javax.management.MBeanInfo;
import javax.management.ReflectionException;
+import java.io.IOException;
import java.lang.reflect.Method;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
/**
@@ -53,6 +59,8 @@ public class RocksDBStoreMBean implements DynamicMBean, MetricsSource {
private Statistics statistics;
+ private final RocksDatabase rocksDB;
+
private Set<String> histogramAttributes = new HashSet<>();
private String contextName;
@@ -61,22 +69,73 @@ public class RocksDBStoreMBean implements DynamicMBean, MetricsSource {
LoggerFactory.getLogger(RocksDBStoreMBean.class);
public static final String ROCKSDB_CONTEXT_PREFIX = "Rocksdb_";
+ public static final String ROCKSDB_PROPERTY_PREFIX = "rocksdb.";
+
+ // RocksDB properties
+ // Column1: rocksDB property original name
+ // Column2: aggregate or not all CF value to get a summed value
+ // Column2: converted rocksDB property name based on Prometheus naming rule
+ private final String[][] cfPros = {
+ // 1 if a memtable flush is pending; otherwise, returns 0
+ {"mem-table-flush-pending", "false", ""},
+ // estimated total number of bytes compaction needs to rewrite to get
+ // all levels down to under target size.
+ {"estimate-pending-compaction-bytes", "true", ""},
+ // 1 if at least one compaction is pending; otherwise, returns 0.
+ {"compaction-pending", "false", ""},
+ // block cache capacity
+ {"block-cache-capacity", "true", ""},
+ // the memory size for the entries residing in block cache
+ {"block-cache-usage", "true", ""},
+ // the memory size for the entries being pinned
+ {"block-cache-pinned-usage", "true", ""},
+ // number of level to which L0 data will be compacted.
+ {"base-level", "false", ""},
+ // approximate active mem table size (bytes)
+ {"cur-size-active-mem-table", "true", ""},
+ // approximate size of active and unflushed immutable (bytes)
+ {"cur-size-all-mem-tables", "true", ""},
+ // approximate size of active, unflushed immutable, and pinned immutable
+ // memtables (bytes)
+ {"size-all-mem-tables", "true", ""},
+ // number of immutable memtables that have not yet been flushed
+ {"num-immutable-mem-table", "true", ""},
+ // total size (bytes) of all SST files belong to the latest LSM tree
+ {"live-sst-files-size", "true", ""},
+ // estimated number of total keys in memtables and storage(Can be very
+ // wrong according to RocksDB document)
+ {"estimate-num-keys", "true", ""},
+ // estimated memory used for reading SST tables, excluding memory used
+ // in block cache (e.g., filter and index blocks)
+ {"estimate-table-readers-mem", "true", ""}
+ };
- public RocksDBStoreMBean(Statistics statistics, String dbName) {
+ // level-x sst file info (Global)
+ private static final String NUM_FILES_AT_LEVEL = "num_files_at_level";
+ private static final String SIZE_AT_LEVEL = "size_at_level";
+
+ public RocksDBStoreMBean(Statistics statistics, RocksDatabase db,
+ String dbName) {
this.contextName = ROCKSDB_CONTEXT_PREFIX + dbName;
this.statistics = statistics;
+ this.rocksDB = db;
histogramAttributes.add("Average");
histogramAttributes.add("Median");
histogramAttributes.add("Percentile95");
histogramAttributes.add("Percentile99");
histogramAttributes.add("StandardDeviation");
+ histogramAttributes.add("Max");
+
+ // To get the metric name complied with prometheus naming rule
+ for (String[] property : cfPros) {
+ property[2] = property[0].replace("-", "_");
+ }
}
public static RocksDBStoreMBean create(Statistics statistics,
- String contextName) {
-
+ RocksDatabase db, String contextName) {
RocksDBStoreMBean rocksDBStoreMBean = new RocksDBStoreMBean(
- statistics, contextName);
+ statistics, db, contextName);
MetricsSystem ms = DefaultMetricsSystem.instance();
MetricsSource metricsSource = ms.getSource(rocksDBStoreMBean.contextName);
if (metricsSource != null) {
@@ -124,7 +183,6 @@ public class RocksDBStoreMBean implements DynamicMBean, MetricsSource {
public void setAttribute(Attribute attribute)
throws AttributeNotFoundException, InvalidAttributeValueException,
MBeanException, ReflectionException {
-
}
@Override
@@ -179,6 +237,7 @@ public class RocksDBStoreMBean implements DynamicMBean, MetricsSource {
MetricsRecordBuilder rb = metricsCollector.addRecord(contextName);
getHistogramData(rb);
getTickerTypeData(rb);
+ getDBPropertyData(rb);
}
/**
@@ -216,4 +275,95 @@ public class RocksDBStoreMBean implements DynamicMBean, MetricsSource {
}
}
+ /**
+ * Collect info from rocksdb property.
+ * @param rb Metrics Record Builder.
+ */
+ private void getDBPropertyData(MetricsRecordBuilder rb) {
+ int index = 0;
+ try {
+ for (index = 0; index < cfPros.length; index++) {
+ Boolean aggregated = Boolean.valueOf(cfPros[index][1]);
+ long sum = 0;
+ for (RocksDatabase.ColumnFamily cf : rocksDB.getExtraColumnFamilies()) {
+ // Metrics per column family
+ long value = Long.parseLong(rocksDB.getProperty(cf,
+ ROCKSDB_PROPERTY_PREFIX + cfPros[index][0]));
+ rb.addCounter(Interns.info(cf.getName() + "_" +
+ cfPros[index][2], "RocksDBProperty"), value);
+ if (aggregated) {
+ sum += value;
+ }
+ }
+ // Export aggregated value
+ if (aggregated) {
+ rb.addCounter(
+ Interns.info(cfPros[index][2], "RocksDBProperty"), sum);
+ }
+ }
+ } catch (IOException e) {
+ LOG.error("Failed to get property {} from rocksdb",
+ cfPros[index][0], e);
+ }
+
+ // Calculate number of files per level and size per level
+ Map<String, Map<Integer, Map<String, Long>>> data = computeSstFileStat();
+
+ // Export file number
+ exportSstFileStat(rb, data.get(NUM_FILES_AT_LEVEL), NUM_FILES_AT_LEVEL);
+
+ // Export file total size
+ exportSstFileStat(rb, data.get(SIZE_AT_LEVEL), SIZE_AT_LEVEL);
+ }
+
+ private Map<String, Map<Integer, Map<String, Long>>> computeSstFileStat() {
+ // Calculate number of files per level and size per level
+ List<LiveFileMetaData> liveFileMetaDataList =
+ rocksDB.getLiveFilesMetaData();
+
+ Map<String, Map<Integer, Map<String, Long>>> ret = new HashMap();
+ Map<Integer, Map<String, Long>> numStatPerCF = new HashMap<>();
+ Map<Integer, Map<String, Long>> sizeStatPerCF = new HashMap<>();
+ Map<String, Long> numStat;
+ Map<String, Long> sizeStat;
+ for (LiveFileMetaData file: liveFileMetaDataList) {
+ numStat = numStatPerCF.get(file.level());
+ String cf = Strings.fromByteArray(file.columnFamilyName());
+ if (numStat != null) {
+ Long value = numStat.get(cf);
+ numStat.put(cf, value == null ? 1L : value + 1);
+ } else {
+ numStat = new HashMap<>();
+ numStat.put(cf, 1L);
+ numStatPerCF.put(file.level(), numStat);
+ }
+
+ sizeStat = sizeStatPerCF.get(file.level());
+ if (sizeStat != null) {
+ Long value = sizeStat.get(cf);
+ sizeStat.put(cf, value == null ? file.size() : value + file.size());
+ } else {
+ sizeStat = new HashMap<>();
+ sizeStat.put(cf, file.size());
+ sizeStatPerCF.put(file.level(), sizeStat);
+ }
+ }
+
+ ret.put(NUM_FILES_AT_LEVEL, numStatPerCF);
+ ret.put(SIZE_AT_LEVEL, sizeStatPerCF);
+ return ret;
+ }
+
+ private void exportSstFileStat(MetricsRecordBuilder rb,
+ Map<Integer, Map<String, Long>> numStatPerCF, String metricName) {
+ Map<String, Long> numStat;
+ for (Map.Entry<Integer, Map<String, Long>> entry: numStatPerCF.entrySet()) {
+ numStat = entry.getValue();
+ numStat.forEach((cf, v) -> rb.addCounter(Interns.info(
+ cf + "_" + metricName + entry.getKey(), "RocksDBProperty"), v));
+ rb.addCounter(
+ Interns.info(metricName + entry.getKey(), "RocksDBProperty"),
+ numStat.values().stream().mapToLong(p -> p.longValue()).sum());
+ }
+ }
}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java
index 1c6c689aeb..d354aedc81 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java
@@ -121,7 +121,8 @@ public class RDBStore implements DBStore {
jmxProperties.put("dbName", dbJmxBeanName);
statMBeanName = HddsUtils.registerWithJmxProperties(
"Ozone", "RocksDbStore", jmxProperties,
- RocksDBStoreMBean.create(dbOptions.statistics(), dbJmxBeanName));
+ RocksDBStoreMBean.create(dbOptions.statistics(), db,
+ dbJmxBeanName));
if (statMBeanName == null) {
LOG.warn("jmx registration failed during RocksDB init, db path :{}",
dbJmxBeanName);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org