You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by sp...@apache.org on 2023/01/16 13:14:09 UTC
[iotdb] branch rel/0.13 updated: [To rel/0.13][IOTDB-5407] metrics for .mods file count (#8874)
This is an automated email from the ASF dual-hosted git repository.
spricoder pushed a commit to branch rel/0.13
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/0.13 by this push:
new 3be148a083 [To rel/0.13][IOTDB-5407] metrics for .mods file count (#8874)
3be148a083 is described below
commit 3be148a083eef04c31b4a13ee411dbd5eea74db7
Author: Pengfei Liu <46...@users.noreply.github.com>
AuthorDate: Mon Jan 16 21:14:00 2023 +0800
[To rel/0.13][IOTDB-5407] metrics for .mods file count (#8874)
* add metrics mods file count
* ListFiles replace org.apache.commons.io.FileUtils, solve the problem not closed
---
.../db/service/metrics/predefined/FileMetrics.java | 102 +++++++++++++++++----
.../java/org/apache/iotdb/db/utils/FileUtils.java | 54 +++++++++++
2 files changed, 138 insertions(+), 18 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/service/metrics/predefined/FileMetrics.java b/server/src/main/java/org/apache/iotdb/db/service/metrics/predefined/FileMetrics.java
index 1a8078ad91..afabbe1797 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/metrics/predefined/FileMetrics.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/metrics/predefined/FileMetrics.java
@@ -49,8 +49,15 @@ public class FileMetrics implements IMetricSet {
private long walFileTotalCount = 0L;
private long sequenceFileTotalSize = 0L;
private long sequenceFileTotalCount = 0L;
+ private long sequenceModsFileTotalCount = 0L;
private long unsequenceFileTotalSize = 0L;
private long unsequenceFileTotalCount = 0L;
+ private long unsequenceModsFileTotalCount = 0L;
+ private static final String KEY_WAL = "wal";
+ private static final String KEY_SEQ = "seq";
+ private static final String KEY_UNSEQ = "unseq";
+ private static final String KEY_SEQ_MODS = "seqmods";
+ private static final String KEY_UNSEQ_MODS = "unseqmods";
public void bindTo(AbstractMetricService metricService) {
metricService.getOrCreateAutoGauge(
@@ -59,42 +66,56 @@ public class FileMetrics implements IMetricSet {
this,
FileMetrics::getWalFileTotalSize,
Tag.NAME.toString(),
- "wal");
+ KEY_WAL);
metricService.getOrCreateAutoGauge(
Metric.FILE_SIZE.toString(),
MetricLevel.IMPORTANT,
this,
FileMetrics::getSequenceFileTotalSize,
Tag.NAME.toString(),
- "seq");
+ KEY_SEQ);
metricService.getOrCreateAutoGauge(
Metric.FILE_SIZE.toString(),
MetricLevel.IMPORTANT,
this,
FileMetrics::getUnsequenceFileTotalSize,
Tag.NAME.toString(),
- "unseq");
+ KEY_UNSEQ);
metricService.getOrCreateAutoGauge(
Metric.FILE_COUNT.toString(),
MetricLevel.IMPORTANT,
this,
FileMetrics::getWalFileTotalCount,
Tag.NAME.toString(),
- "wal");
+ KEY_WAL);
metricService.getOrCreateAutoGauge(
Metric.FILE_COUNT.toString(),
MetricLevel.IMPORTANT,
this,
FileMetrics::getSequenceFileTotalCount,
Tag.NAME.toString(),
- "seq");
+ KEY_SEQ);
metricService.getOrCreateAutoGauge(
Metric.FILE_COUNT.toString(),
MetricLevel.IMPORTANT,
this,
FileMetrics::getUnsequenceFileTotalCount,
Tag.NAME.toString(),
- "unseq");
+ KEY_UNSEQ);
+ metricService.getOrCreateAutoGauge(
+ Metric.FILE_COUNT.toString(),
+ MetricLevel.IMPORTANT,
+ this,
+ FileMetrics::getSequenceModsFileTotalCount,
+ Tag.NAME.toString(),
+ KEY_SEQ_MODS);
+ metricService.getOrCreateAutoGauge(
+ Metric.FILE_COUNT.toString(),
+ MetricLevel.IMPORTANT,
+ this,
+ FileMetrics::getUnsequenceModsFileTotalCount,
+ Tag.NAME.toString(),
+ KEY_UNSEQ_MODS);
// finally start to update the value of some metrics in async way
if (metricService.isEnable() && null == currentServiceFuture) {
@@ -117,16 +138,22 @@ public class FileMetrics implements IMetricSet {
currentServiceFuture = null;
}
- metricService.remove(MetricType.GAUGE, Metric.FILE_SIZE.toString(), Tag.NAME.toString(), "wal");
- metricService.remove(MetricType.GAUGE, Metric.FILE_SIZE.toString(), Tag.NAME.toString(), "seq");
metricService.remove(
- MetricType.GAUGE, Metric.FILE_SIZE.toString(), Tag.NAME.toString(), "unseq");
+ MetricType.GAUGE, Metric.FILE_SIZE.toString(), Tag.NAME.toString(), KEY_WAL);
+ metricService.remove(
+ MetricType.GAUGE, Metric.FILE_SIZE.toString(), Tag.NAME.toString(), KEY_SEQ);
+ metricService.remove(
+ MetricType.GAUGE, Metric.FILE_SIZE.toString(), Tag.NAME.toString(), KEY_UNSEQ);
metricService.remove(
- MetricType.GAUGE, Metric.FILE_COUNT.toString(), Tag.NAME.toString(), "wal");
+ MetricType.GAUGE, Metric.FILE_COUNT.toString(), Tag.NAME.toString(), KEY_WAL);
metricService.remove(
- MetricType.GAUGE, Metric.FILE_COUNT.toString(), Tag.NAME.toString(), "seq");
+ MetricType.GAUGE, Metric.FILE_COUNT.toString(), Tag.NAME.toString(), KEY_SEQ);
metricService.remove(
- MetricType.GAUGE, Metric.FILE_COUNT.toString(), Tag.NAME.toString(), "unseq");
+ MetricType.GAUGE, Metric.FILE_COUNT.toString(), Tag.NAME.toString(), KEY_UNSEQ);
+ metricService.remove(
+ MetricType.GAUGE, Metric.FILE_COUNT.toString(), Tag.NAME.toString(), KEY_SEQ_MODS);
+ metricService.remove(
+ MetricType.GAUGE, Metric.FILE_COUNT.toString(), Tag.NAME.toString(), KEY_UNSEQ_MODS);
}
private void collect() {
@@ -151,8 +178,7 @@ public class FileMetrics implements IMetricSet {
.sum();
File walFolder = new File(walDirs);
if (walFolder.exists() && walFolder.isDirectory()) {
- walFileTotalCount =
- org.apache.commons.io.FileUtils.listFiles(new File(walDirs), null, true).size();
+ walFileTotalCount = FileUtils.listFiles(new File(walDirs), null, true).size();
}
sequenceFileTotalCount =
Stream.of(dataDirs)
@@ -162,8 +188,7 @@ public class FileMetrics implements IMetricSet {
File folder = new File(dir);
if (folder.exists()) {
try {
- return org.apache.commons.io.FileUtils.listFiles(
- new File(dir), new String[] {"tsfile"}, true)
+ return FileUtils.listFiles(new File(dir), new String[] {"tsfile"}, true)
.size();
} catch (UncheckedIOException exception) {
// do nothing
@@ -173,6 +198,23 @@ public class FileMetrics implements IMetricSet {
return 0L;
})
.sum();
+ sequenceModsFileTotalCount =
+ Stream.of(dataDirs)
+ .mapToLong(
+ dir -> {
+ dir += File.separator + IoTDBConstant.SEQUENCE_FLODER_NAME;
+ File folder = new File(dir);
+ if (folder.exists()) {
+ try {
+ return FileUtils.listFiles(new File(dir), new String[] {"mods"}, true).size();
+ } catch (UncheckedIOException exception) {
+ // do nothing
+ logger.debug("Failed when count sequence mods: ", exception);
+ }
+ }
+ return 0L;
+ })
+ .sum();
unsequenceFileTotalCount =
Stream.of(dataDirs)
.mapToLong(
@@ -181,8 +223,7 @@ public class FileMetrics implements IMetricSet {
File folder = new File(dir);
if (folder.exists()) {
try {
- return org.apache.commons.io.FileUtils.listFiles(
- new File(dir), new String[] {"tsfile"}, true)
+ return FileUtils.listFiles(new File(dir), new String[] {"tsfile"}, true)
.size();
} catch (UncheckedIOException exception) {
// do nothing
@@ -192,6 +233,23 @@ public class FileMetrics implements IMetricSet {
return 0L;
})
.sum();
+ unsequenceModsFileTotalCount =
+ Stream.of(dataDirs)
+ .mapToLong(
+ dir -> {
+ dir += File.separator + IoTDBConstant.UNSEQUENCE_FLODER_NAME;
+ File folder = new File(dir);
+ if (folder.exists()) {
+ try {
+ return FileUtils.listFiles(new File(dir), new String[] {"mods"}, true).size();
+ } catch (UncheckedIOException exception) {
+ // do nothing
+ logger.debug("Failed when count unsequence mods: ", exception);
+ }
+ }
+ return 0L;
+ })
+ .sum();
}
public long getWalFileTotalSize() {
@@ -210,6 +268,10 @@ public class FileMetrics implements IMetricSet {
return sequenceFileTotalCount;
}
+ private long getSequenceModsFileTotalCount() {
+ return sequenceModsFileTotalCount;
+ }
+
public long getUnsequenceFileTotalSize() {
return unsequenceFileTotalSize;
}
@@ -217,4 +279,8 @@ public class FileMetrics implements IMetricSet {
public long getUnsequenceFileTotalCount() {
return unsequenceFileTotalCount;
}
+
+ private long getUnsequenceModsFileTotalCount() {
+ return unsequenceModsFileTotalCount;
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/FileUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/FileUtils.java
index 1912f22172..9496b026b1 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/FileUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/FileUtils.java
@@ -20,14 +20,26 @@ package org.apache.iotdb.db.utils;
import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
+import org.apache.commons.io.filefilter.FileFileFilter;
+import org.apache.commons.io.filefilter.IOFileFilter;
+import org.apache.commons.io.filefilter.SuffixFileFilter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
+import java.io.IOException;
+import java.io.UncheckedIOException;
import java.nio.file.DirectoryNotEmptyException;
+import java.nio.file.FileVisitOption;
+import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
+import java.nio.file.Path;
import java.util.Arrays;
+import java.util.Collection;
+import java.util.Objects;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
public class FileUtils {
private static Logger logger = LoggerFactory.getLogger(FileUtils.class);
@@ -70,4 +82,46 @@ public class FileUtils {
}
return sum;
}
+
+ /**
+ * ListFiles replace org.apache.commons.io.FileUtils, solve the problem not closed
+ *
+ * @param directory
+ * @param extensions
+ * @param recursive
+ * @return
+ * @throws IOException
+ */
+ public static Collection<File> listFiles(File directory, String[] extensions, boolean recursive) {
+ if (directory == null) {
+ throw new UncheckedIOException(new IOException("directory is null"));
+ }
+ IOFileFilter pathFilter =
+ extensions == null
+ ? FileFileFilter.INSTANCE
+ : FileFileFilter.INSTANCE.and(new SuffixFileFilter(toSuffixes(extensions)));
+ try (Stream<Path> s =
+ Files.walk(directory.toPath(), toMaxDepth(recursive), FileVisitOption.FOLLOW_LINKS)) {
+ return s.filter((path) -> pathFilter.accept(path, null) == FileVisitResult.CONTINUE)
+ .map(Path::toFile)
+ .collect(Collectors.toList());
+ } catch (IOException e) {
+ throw new UncheckedIOException(directory.toString(), e);
+ }
+ }
+
+ private static String[] toSuffixes(String... extensions) {
+ Objects.requireNonNull(extensions, "extensions");
+ String[] suffixes = new String[extensions.length];
+
+ for (int i = 0; i < extensions.length; ++i) {
+ suffixes[i] = "." + extensions[i];
+ }
+
+ return suffixes;
+ }
+
+ private static int toMaxDepth(boolean recursive) {
+ return recursive ? 2147483647 : 1;
+ }
}