You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tsfile.apache.org by ji...@apache.org on 2026/07/29 10:41:38 UTC
(tsfile) branch add_property_read_write_interface created (now 95247656f)
This is an automated email from the ASF dual-hosted git repository.
jt2594838 pushed a change to branch add_property_read_write_interface
in repository https://gitbox.apache.org/repos/asf/tsfile.git
at 95247656f Add perf test
This branch includes the following new commits:
new 91734b313 add property read-write interface
new 395e57a19 May record table point count
new 95247656f Add perf test
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(tsfile) 03/03: Add perf test
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jt2594838 pushed a commit to branch add_property_read_write_interface
in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit 95247656f01db2e165fe2d2fb07e06ccd29344cb
Author: Tian Jiang <jt...@163.com>
AuthorDate: Wed Jul 29 18:40:31 2026 +0800
Add perf test
---
.../write/TablePointCountPerformanceTest.java | 266 +++++++++++++++++++++
1 file changed, 266 insertions(+)
diff --git a/java/tsfile/src/test/java/org/apache/tsfile/write/TablePointCountPerformanceTest.java b/java/tsfile/src/test/java/org/apache/tsfile/write/TablePointCountPerformanceTest.java
new file mode 100644
index 000000000..e31c4fd44
--- /dev/null
+++ b/java/tsfile/src/test/java/org/apache/tsfile/write/TablePointCountPerformanceTest.java
@@ -0,0 +1,266 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tsfile.write;
+
+import org.apache.tsfile.enums.ColumnCategory;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.exception.write.WriteProcessException;
+import org.apache.tsfile.file.metadata.ColumnSchema;
+import org.apache.tsfile.file.metadata.TableSchema;
+import org.apache.tsfile.read.TsFileSequenceReader;
+import org.apache.tsfile.write.record.Tablet;
+import org.apache.tsfile.write.schema.IMeasurementSchema;
+import org.apache.tsfile.write.writer.TsFileIOWriter;
+
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+public class TablePointCountPerformanceTest {
+
+ private static final String MODE_PROPERTY = "tablePointCountEnabled";
+ private static final int TABLE1_ROW_COUNT = 200_000;
+ private static final int TABLE2_ROW_COUNT = 100_000;
+ private static final int BATCH_SIZE = 100;
+ private static final long TABLE1_POINT_COUNT = 333_333;
+ private static final long TABLE2_POINT_COUNT = 100_000;
+
+ /**
+ * Measures one table point-count mode per JVM. Invoke this test twice with {@code
+ * -DtablePointCountEnabled=false} and {@code true}; each JVM performs its own warm-up and writes
+ * a mode-specific metrics file. Once both files exist, the test generates a comparison report
+ * containing the two JVM process IDs, file sizes, and measured write durations.
+ */
+ @Test
+ public void measureTablePointCountInDedicatedJvm() throws IOException, WriteProcessException {
+ String configuredMode = System.getProperty(MODE_PROPERTY);
+ Assume.assumeTrue(
+ "Performance test requires -D" + MODE_PROPERTY + "=true|false", configuredMode != null);
+ Assert.assertTrue(
+ "Invalid -D" + MODE_PROPERTY + " value: " + configuredMode,
+ "true".equals(configuredMode) || "false".equals(configuredMode));
+ boolean enabled = Boolean.parseBoolean(configuredMode);
+
+ TableSchema tableSchema1 =
+ new TableSchema(
+ "table1",
+ Arrays.asList(
+ new ColumnSchema("device", TSDataType.STRING, ColumnCategory.TAG),
+ new ColumnSchema("s1", TSDataType.INT32, ColumnCategory.FIELD),
+ new ColumnSchema("s2", TSDataType.INT32, ColumnCategory.FIELD)));
+ TableSchema tableSchema2 =
+ new TableSchema(
+ "table2",
+ Arrays.asList(
+ new ColumnSchema("device", TSDataType.STRING, ColumnCategory.TAG),
+ new ColumnSchema("s1", TSDataType.INT32, ColumnCategory.FIELD)));
+ List<Tablet> table1Tablets = createTablets(tableSchema1, TABLE1_ROW_COUNT, true);
+ List<Tablet> table2Tablets = createTablets(tableSchema2, TABLE2_ROW_COUNT, false);
+
+ File targetDirectory = new File("target");
+ File warmUpFile = new File(targetDirectory, "table-point-count-warm-up-" + enabled + ".tsfile");
+ File measuredFile = new File(targetDirectory, "table-point-count-" + enabled + ".tsfile");
+ try {
+ writeFile(warmUpFile, enabled, tableSchema1, tableSchema2, table1Tablets, table2Tablets);
+ Assert.assertTrue(warmUpFile.delete());
+
+ long startNanos = System.nanoTime();
+ writeFile(measuredFile, enabled, tableSchema1, tableSchema2, table1Tablets, table2Tablets);
+ long elapsedNanos = System.nanoTime() - startNanos;
+ Assert.assertTrue(elapsedNanos > 0);
+
+ verifyProperties(measuredFile, enabled);
+ long pid = ProcessHandle.current().pid();
+ writeMetrics(targetDirectory, enabled, pid, measuredFile.length(), elapsedNanos);
+ generateReportIfBothModesExist(targetDirectory);
+ System.out.printf(
+ "Table point count enabled=%s: pid=%d, size=%d bytes, write=%d ns (%.3f ms)%n",
+ enabled, pid, measuredFile.length(), elapsedNanos, elapsedNanos / 1_000_000.0);
+ } finally {
+ warmUpFile.delete();
+ measuredFile.delete();
+ }
+ }
+
+ private void writeFile(
+ File file,
+ boolean enabled,
+ TableSchema tableSchema1,
+ TableSchema tableSchema2,
+ List<Tablet> table1Tablets,
+ List<Tablet> table2Tablets)
+ throws IOException, WriteProcessException {
+ try (TsFileWriter writer = new TsFileWriter(new TsFileIOWriter(file, enabled))) {
+ writer.registerTableSchema(tableSchema1);
+ writer.registerTableSchema(tableSchema2);
+ for (Tablet tablet : table1Tablets) {
+ writer.writeTable(tablet);
+ }
+ for (Tablet tablet : table2Tablets) {
+ writer.writeTable(tablet);
+ }
+ }
+ }
+
+ private List<Tablet> createTablets(
+ TableSchema tableSchema, int rowCount, boolean nullableSecondField) {
+ List<String> columnNames =
+ IMeasurementSchema.getMeasurementNameList(tableSchema.getColumnSchemas());
+ List<TSDataType> dataTypes = IMeasurementSchema.getDataTypeList(tableSchema.getColumnSchemas());
+ List<Tablet> tablets = new ArrayList<>((rowCount + BATCH_SIZE - 1) / BATCH_SIZE);
+ for (int startRow = 0; startRow < rowCount; startRow += BATCH_SIZE) {
+ int currentBatchSize = Math.min(BATCH_SIZE, rowCount - startRow);
+ Tablet tablet =
+ new Tablet(
+ tableSchema.getTableName(),
+ columnNames,
+ dataTypes,
+ tableSchema.getColumnTypes(),
+ currentBatchSize);
+ for (int row = 0; row < currentBatchSize; row++) {
+ int globalRow = startRow + row;
+ tablet.addTimestamp(row, globalRow);
+ tablet.addValue("device", row, "d1");
+ tablet.addValue("s1", row, globalRow);
+ if (nullableSecondField) {
+ tablet.addValue("s2", row, globalRow % 3 == 0 ? null : globalRow);
+ }
+ }
+ tablets.add(tablet);
+ }
+ return tablets;
+ }
+
+ private void verifyProperties(File file, boolean enabled) throws IOException {
+ try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getAbsolutePath())) {
+ Map<String, String> properties = reader.getTsFileProperties();
+ String table1Key = TsFileIOWriter.TABLE_POINT_COUNT_PROPERTY_PREFIX + "table1";
+ String table2Key = TsFileIOWriter.TABLE_POINT_COUNT_PROPERTY_PREFIX + "table2";
+ if (enabled) {
+ Assert.assertEquals(Long.toString(TABLE1_POINT_COUNT), properties.get(table1Key));
+ Assert.assertEquals(Long.toString(TABLE2_POINT_COUNT), properties.get(table2Key));
+ } else {
+ Assert.assertFalse(properties.containsKey(table1Key));
+ Assert.assertFalse(properties.containsKey(table2Key));
+ }
+ }
+ }
+
+ private void writeMetrics(
+ File targetDirectory, boolean enabled, long pid, long fileSize, long elapsedNanos)
+ throws IOException {
+ Properties metrics = new Properties();
+ metrics.setProperty("enabled", Boolean.toString(enabled));
+ metrics.setProperty("pid", Long.toString(pid));
+ metrics.setProperty("fileSize", Long.toString(fileSize));
+ metrics.setProperty("elapsedNanos", Long.toString(elapsedNanos));
+ metrics.setProperty("rows", Integer.toString(TABLE1_ROW_COUNT + TABLE2_ROW_COUNT));
+ metrics.setProperty("points", Long.toString(TABLE1_POINT_COUNT + TABLE2_POINT_COUNT));
+ try (FileOutputStream output = new FileOutputStream(metricsFile(targetDirectory, enabled))) {
+ metrics.store(output, "Table point count performance metrics");
+ }
+ }
+
+ private void generateReportIfBothModesExist(File targetDirectory) throws IOException {
+ File disabledMetricsFile = metricsFile(targetDirectory, false);
+ File enabledMetricsFile = metricsFile(targetDirectory, true);
+ if (!disabledMetricsFile.isFile() || !enabledMetricsFile.isFile()) {
+ return;
+ }
+ Properties disabled = loadMetrics(disabledMetricsFile);
+ Properties enabled = loadMetrics(enabledMetricsFile);
+ long disabledPid = Long.parseLong(disabled.getProperty("pid"));
+ long enabledPid = Long.parseLong(enabled.getProperty("pid"));
+ Assert.assertNotEquals("Modes must run in separate JVMs", disabledPid, enabledPid);
+ long disabledSize = Long.parseLong(disabled.getProperty("fileSize"));
+ long enabledSize = Long.parseLong(enabled.getProperty("fileSize"));
+ long disabledNanos = Long.parseLong(disabled.getProperty("elapsedNanos"));
+ long enabledNanos = Long.parseLong(enabled.getProperty("elapsedNanos"));
+ Assert.assertTrue(enabledSize > disabledSize);
+
+ String report =
+ String.format(
+ "# Table Point Count Performance Report%n%n"
+ + "## Dataset%n%n"
+ + "- table1: %,d rows, %,d non-null FIELD points%n"
+ + "- table2: %,d rows, %,d non-null FIELD points%n"
+ + "- Batch size: %,d rows%n"
+ + "- Total: %,d rows, %,d non-null FIELD points%n%n"
+ + "## Results%n%n"
+ + "| Point counting | JVM PID | File size (bytes) | Write time (ns) | Write time (ms) |%n"
+ + "|---|---:|---:|---:|---:|%n"
+ + "| Disabled | %d | %,d | %,d | %.3f |%n"
+ + "| Enabled | %d | %,d | %,d | %.3f |%n%n"
+ + "- File-size delta: %,d bytes%n"
+ + "- File-size overhead: %.3f%%%n"
+ + "- Write-time delta: %,d ns (%.3f ms)%n"
+ + "- Write-time overhead: %.3f%%%n%n"
+ + "> Each mode ran in a separate JVM and performed an independent warm-up. Write "
+ + "time includes writer construction, schema registration, batched writes, "
+ + "metadata serialization, force, and close. Timing is diagnostic and is not used "
+ + "as a CI pass/fail threshold.%n",
+ TABLE1_ROW_COUNT,
+ TABLE1_POINT_COUNT,
+ TABLE2_ROW_COUNT,
+ TABLE2_POINT_COUNT,
+ BATCH_SIZE,
+ TABLE1_ROW_COUNT + TABLE2_ROW_COUNT,
+ TABLE1_POINT_COUNT + TABLE2_POINT_COUNT,
+ disabledPid,
+ disabledSize,
+ disabledNanos,
+ disabledNanos / 1_000_000.0,
+ enabledPid,
+ enabledSize,
+ enabledNanos,
+ enabledNanos / 1_000_000.0,
+ enabledSize - disabledSize,
+ (enabledSize - disabledSize) * 100.0 / disabledSize,
+ enabledNanos - disabledNanos,
+ (enabledNanos - disabledNanos) / 1_000_000.0,
+ (enabledNanos - disabledNanos) * 100.0 / disabledNanos);
+ File reportFile = new File(targetDirectory, "table-point-count-performance-report.md");
+ Files.write(reportFile.toPath(), report.getBytes(StandardCharsets.UTF_8));
+ System.out.printf("Table point-count report: %s%n", reportFile.getAbsolutePath());
+ }
+
+ private Properties loadMetrics(File file) throws IOException {
+ Properties metrics = new Properties();
+ try (FileInputStream input = new FileInputStream(file)) {
+ metrics.load(input);
+ }
+ return metrics;
+ }
+
+ private File metricsFile(File targetDirectory, boolean enabled) {
+ return new File(targetDirectory, "table-point-count-" + enabled + ".properties");
+ }
+}
(tsfile) 01/03: add property read-write interface
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jt2594838 pushed a commit to branch add_property_read_write_interface
in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit 91734b313ae3917c44a0dd516785f5ee423eac5b
Author: Tian Jiang <jt...@163.com>
AuthorDate: Wed Jul 29 12:17:14 2026 +0800
add property read-write interface
---
.../apache/tsfile/read/TsFileSequenceReader.java | 5 ++++
.../tsfile/read/v4/DeviceTableModelReader.java | 6 ++++
.../org/apache/tsfile/read/v4/ITsFileReader.java | 4 +++
.../write/v4/AbstractTableModelTsFileWriter.java | 6 ++++
.../org/apache/tsfile/write/v4/ITsFileWriter.java | 3 ++
.../apache/tsfile/write/writer/TsFileIOWriter.java | 8 ++++++
.../read/TsFileV4ReadWriteInterfacesTest.java | 33 ++++++++++++++++++++++
7 files changed, 65 insertions(+)
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java
index 1fcabdab6..505b6aa8a 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java
@@ -607,6 +607,11 @@ public class TsFileSequenceReader implements AutoCloseable {
return getTableSchemaMap(null);
}
+ /** Get the properties stored in the TsFile metadata. */
+ public Map<String, String> getTsFileProperties() throws IOException {
+ return readFileMetadata().getTsFileProperties();
+ }
+
public Map<String, TableSchema> getTableSchemaMap(LongConsumer ioSizeRecorder)
throws IOException {
if (tsFileMetaData != null && tsFileMetaData.hasTableSchemaMapCache()) {
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/v4/DeviceTableModelReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/v4/DeviceTableModelReader.java
index e7749d42b..4dffbbc01 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/v4/DeviceTableModelReader.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/v4/DeviceTableModelReader.java
@@ -78,6 +78,12 @@ public class DeviceTableModelReader implements ITsFileReader {
return Optional.ofNullable(tableSchemaMap.get(tableName.toLowerCase()));
}
+ @Override
+ @TsFileApi
+ public Map<String, String> getTsFileProperties() throws IOException {
+ return fileReader.getTsFileProperties();
+ }
+
@TsFileApi
public ResultSet query(String tableName, List<String> columnNames, long startTime, long endTime)
throws IOException, NoTableException, NoMeasurementException, ReadProcessException {
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/v4/ITsFileReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/v4/ITsFileReader.java
index 995f73f64..0ade853c7 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/v4/ITsFileReader.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/v4/ITsFileReader.java
@@ -29,6 +29,7 @@ import org.apache.tsfile.read.query.dataset.ResultSet;
import java.io.IOException;
import java.util.List;
+import java.util.Map;
import java.util.Optional;
public interface ITsFileReader extends AutoCloseable {
@@ -48,6 +49,9 @@ public interface ITsFileReader extends AutoCloseable {
@TsFileApi
List<TableSchema> getAllTableSchema() throws IOException;
+ @TsFileApi
+ Map<String, String> getTsFileProperties() throws IOException;
+
@TsFileApi
void close();
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/AbstractTableModelTsFileWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/AbstractTableModelTsFileWriter.java
index 94be1fdc1..2bb39ad57 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/AbstractTableModelTsFileWriter.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/AbstractTableModelTsFileWriter.java
@@ -259,6 +259,12 @@ abstract class AbstractTableModelTsFileWriter implements ITsFileWriter {
return fileWriter.getSchema();
}
+ @Override
+ @TsFileApi
+ public void addTsFileProperty(String key, String value) {
+ fileWriter.addTsFileProperty(key, value);
+ }
+
/**
* calling this method to write the last data remaining in memory and close the normal and error
* OutputStream.
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/ITsFileWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/ITsFileWriter.java
index 25ccb031b..15e5db30b 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/ITsFileWriter.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/ITsFileWriter.java
@@ -36,4 +36,7 @@ public interface ITsFileWriter extends AutoCloseable {
@TsFileApi
void write(TSRecord record) throws IOException, WriteProcessException;
+
+ @TsFileApi
+ void addTsFileProperty(String key, String value);
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java
index df6d99b20..a379e4314 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java
@@ -143,6 +143,8 @@ public class TsFileIOWriter implements AutoCloseable {
protected Map<String, Long> tableSizeMap = new HashMap<>();
+ private final Map<String, String> tsFileProperties = new HashMap<>();
+
/** empty construct function. */
protected TsFileIOWriter() {
setEncryptParam(
@@ -246,6 +248,11 @@ public class TsFileIOWriter implements AutoCloseable {
}
}
+ /** Add a custom property to the TsFile metadata. */
+ public void addTsFileProperty(String key, String value) {
+ tsFileProperties.put(key, value);
+ }
+
public void addFlushListener(FlushChunkMetadataListener listener) {
flushListeners.add(listener);
}
@@ -590,6 +597,7 @@ public class TsFileIOWriter implements AutoCloseable {
tsFileMetadata.setTableSchemaMap(schema.getTableSchemaMap());
tsFileMetadata.setMetaOffset(metaOffset);
tsFileMetadata.setBloomFilter(filter);
+ tsFileProperties.forEach(tsFileMetadata::addProperty);
tsFileMetadata.addProperty("encryptLevel", encryptLevel);
tsFileMetadata.addProperty("encryptType", encryptType);
tsFileMetadata.addProperty("encryptKey", encryptKey);
diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileV4ReadWriteInterfacesTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileV4ReadWriteInterfacesTest.java
index 51aa64ce6..c8b0b5c0d 100644
--- a/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileV4ReadWriteInterfacesTest.java
+++ b/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileV4ReadWriteInterfacesTest.java
@@ -27,6 +27,7 @@ import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.file.metadata.StringArrayDeviceID;
import org.apache.tsfile.file.metadata.TableSchema;
import org.apache.tsfile.read.v4.DeviceTableModelReader;
+import org.apache.tsfile.read.v4.ITsFileReader;
import org.apache.tsfile.utils.Pair;
import org.apache.tsfile.utils.TsFileGeneratorForTest;
import org.apache.tsfile.utils.TsFileGeneratorUtils;
@@ -49,6 +50,38 @@ import java.util.List;
public class TsFileV4ReadWriteInterfacesTest {
+ @Test
+ public void testTsFileProperties() throws IOException {
+ String filePath = TsFileGeneratorForTest.getTestTsFilePath("properties", 0, 0, 0);
+ File file = new File(filePath);
+ TableSchema tableSchema =
+ new TableSchema(
+ "t1",
+ Arrays.asList(new MeasurementSchema("s1", TSDataType.INT32)),
+ Arrays.asList(ColumnCategory.FIELD));
+
+ try {
+ try (ITsFileWriter writer =
+ new TsFileWriterBuilder().file(file).tableSchema(tableSchema).build()) {
+ writer.addTsFileProperty("creator", "TsFileV4ReadWriteInterfacesTest");
+ writer.addTsFileProperty("version", "1");
+ }
+
+ try (TsFileSequenceReader reader = new TsFileSequenceReader(filePath)) {
+ Assert.assertEquals(
+ "TsFileV4ReadWriteInterfacesTest", reader.getTsFileProperties().get("creator"));
+ Assert.assertEquals("1", reader.getTsFileProperties().get("version"));
+ }
+ try (ITsFileReader reader = new DeviceTableModelReader(file)) {
+ Assert.assertEquals(
+ "TsFileV4ReadWriteInterfacesTest", reader.getTsFileProperties().get("creator"));
+ Assert.assertEquals("1", reader.getTsFileProperties().get("version"));
+ }
+ } finally {
+ Files.deleteIfExists(file.toPath());
+ }
+ }
+
@Test
public void testWriteSomeColumns() throws IOException, WriteProcessException {
String filePath = TsFileGeneratorForTest.getTestTsFilePath("db", 0, 0, 0);
(tsfile) 02/03: May record table point count
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jt2594838 pushed a commit to branch add_property_read_write_interface
in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit 395e57a19c3a6c71be055ced50d6ebbec5d16aa5
Author: Tian Jiang <jt...@163.com>
AuthorDate: Wed Jul 29 17:46:57 2026 +0800
May record table point count
---
.../apache/tsfile/write/writer/TsFileIOWriter.java | 32 ++++++++++++
.../apache/tsfile/write/TsFileWriteApiTest.java | 61 ++++++++++++++++++++++
2 files changed, 93 insertions(+)
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java
index a379e4314..79034b28e 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java
@@ -126,6 +126,7 @@ public class TsFileIOWriter implements AutoCloseable {
protected LinkedList<Long> endPosInCMTForDevice = new LinkedList<>();
private volatile int chunkMetadataCount = 0;
public static final String CHUNK_METADATA_TEMP_FILE_SUFFIX = ".meta";
+ public static final String TABLE_POINT_COUNT_PROPERTY_PREFIX = "tablePointCount.";
private boolean generateTableSchema = false;
@@ -144,6 +145,8 @@ public class TsFileIOWriter implements AutoCloseable {
protected Map<String, Long> tableSizeMap = new HashMap<>();
private final Map<String, String> tsFileProperties = new HashMap<>();
+ private final Map<String, Long> tablePointCountMap = new HashMap<>();
+ private boolean recordTablePointCount;
/** empty construct function. */
protected TsFileIOWriter() {
@@ -165,6 +168,15 @@ public class TsFileIOWriter implements AutoCloseable {
this(file, TS_FILE_CONFIG);
}
+ /**
+ * Creates a writer and optionally records the number of non-null field points for each table in
+ * the TsFile properties.
+ */
+ public TsFileIOWriter(File file, boolean recordTablePointCount) throws IOException {
+ this(file);
+ this.recordTablePointCount = recordTablePointCount;
+ }
+
public TsFileIOWriter(File file, EncryptParameter param) throws IOException {
this(file, TS_FILE_CONFIG, param);
}
@@ -368,6 +380,7 @@ public class TsFileIOWriter implements AutoCloseable {
chunkHeader.getCompressionType(),
out.getPosition(),
chunkMetadata.getStatistics());
+ currentChunkMetadata.setMask(chunkMetadata.getMask());
chunkHeader.serializeTo(out.wrapAsStream());
out.write(chunk.getData());
endCurrentChunk();
@@ -419,6 +432,10 @@ public class TsFileIOWriter implements AutoCloseable {
chunkHeader.getCompressionType(),
out.getPosition(),
chunk.getChunkStatistic());
+ currentChunkMetadata.setMask(
+ (byte)
+ (chunkHeader.getChunkType()
+ & (TsFileConstant.TIME_COLUMN_MASK | TsFileConstant.VALUE_COLUMN_MASK)));
chunkHeader.serializeTo(out.wrapAsStream());
out.write(chunk.getData());
endCurrentChunk();
@@ -428,6 +445,15 @@ public class TsFileIOWriter implements AutoCloseable {
public void endCurrentChunk() {
this.currentChunkMetadataSize += currentChunkMetadata.getRetainedSizeInBytes();
chunkMetadataCount++;
+ if (recordTablePointCount
+ && currentChunkGroupDeviceId != null
+ && currentChunkGroupDeviceId.isTableModel()
+ && (currentChunkMetadata.getMask() & TsFileConstant.TIME_COLUMN_MASK) == 0) {
+ tablePointCountMap.merge(
+ currentChunkGroupDeviceId.getTableName(),
+ currentChunkMetadata.getNumOfPoints(),
+ Long::sum);
+ }
chunkMetadataList.add(currentChunkMetadata);
currentChunkMetadata = null;
}
@@ -598,6 +624,12 @@ public class TsFileIOWriter implements AutoCloseable {
tsFileMetadata.setMetaOffset(metaOffset);
tsFileMetadata.setBloomFilter(filter);
tsFileProperties.forEach(tsFileMetadata::addProperty);
+ if (recordTablePointCount) {
+ tablePointCountMap.forEach(
+ (tableName, pointCount) ->
+ tsFileMetadata.addProperty(
+ TABLE_POINT_COUNT_PROPERTY_PREFIX + tableName, Long.toString(pointCount)));
+ }
tsFileMetadata.addProperty("encryptLevel", encryptLevel);
tsFileMetadata.addProperty("encryptType", encryptType);
tsFileMetadata.addProperty("encryptKey", encryptKey);
diff --git a/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java b/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java
index fcd761d3f..c8592afe1 100644
--- a/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java
+++ b/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java
@@ -1237,6 +1237,67 @@ public class TsFileWriteApiTest {
}
}
+ /**
+ * Verifies that table point counting records only non-null FIELD values in TsFile properties.
+ * Table1 contains three s1 values and two non-null s2 values, while table2 contains two s1
+ * values. Timestamps, TAG values, and the null s2 value are excluded, so the expected counts are
+ * five and two respectively.
+ */
+ @Test
+ public void recordTablePointCountInProperties() throws IOException, WriteProcessException {
+ TableSchema tableSchema1 =
+ new TableSchema(
+ "table1",
+ Arrays.asList(
+ new ColumnSchema("device", TSDataType.STRING, ColumnCategory.TAG),
+ new ColumnSchema("s1", TSDataType.INT32, ColumnCategory.FIELD),
+ new ColumnSchema("s2", TSDataType.INT32, ColumnCategory.FIELD)));
+ TableSchema tableSchema2 =
+ new TableSchema(
+ "table2",
+ Arrays.asList(
+ new ColumnSchema("device", TSDataType.STRING, ColumnCategory.TAG),
+ new ColumnSchema("s1", TSDataType.INT32, ColumnCategory.FIELD)));
+ Tablet tablet1 =
+ new Tablet(
+ "table1",
+ IMeasurementSchema.getMeasurementNameList(tableSchema1.getColumnSchemas()),
+ IMeasurementSchema.getDataTypeList(tableSchema1.getColumnSchemas()),
+ tableSchema1.getColumnTypes());
+ for (int row = 0; row < 3; row++) {
+ tablet1.addTimestamp(row, row);
+ tablet1.addValue("device", row, "d1");
+ tablet1.addValue("s1", row, row);
+ tablet1.addValue("s2", row, row == 2 ? null : row);
+ }
+ Tablet tablet2 =
+ new Tablet(
+ "table2",
+ IMeasurementSchema.getMeasurementNameList(tableSchema2.getColumnSchemas()),
+ IMeasurementSchema.getDataTypeList(tableSchema2.getColumnSchemas()),
+ tableSchema2.getColumnTypes());
+ for (int row = 0; row < 2; row++) {
+ tablet2.addTimestamp(row, row);
+ tablet2.addValue("device", row, "d1");
+ tablet2.addValue("s1", row, row);
+ }
+
+ try (TsFileWriter writer = new TsFileWriter(new TsFileIOWriter(f, true))) {
+ writer.registerTableSchema(tableSchema1);
+ writer.registerTableSchema(tableSchema2);
+ writer.writeTable(tablet1);
+ writer.writeTable(tablet2);
+ }
+
+ try (TsFileSequenceReader reader = new TsFileSequenceReader(f.getAbsolutePath())) {
+ Map<String, String> properties = reader.getTsFileProperties();
+ Assert.assertEquals(
+ "5", properties.get(TsFileIOWriter.TABLE_POINT_COUNT_PROPERTY_PREFIX + "table1"));
+ Assert.assertEquals(
+ "2", properties.get(TsFileIOWriter.TABLE_POINT_COUNT_PROPERTY_PREFIX + "table2"));
+ }
+ }
+
@Test
public void calculateTableSize() throws IOException, WriteProcessException {
TableSchema tableSchema1 =