You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tsfile.apache.org by ji...@apache.org on 2025/10/28 01:41:35 UTC
(tsfile) branch develop updated: New java interfaces (#602)
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/tsfile.git
The following commit(s) were added to refs/heads/develop by this push:
new cad87cb1 New java interfaces (#602)
cad87cb1 is described below
commit cad87cb12b97eb465de834eb460a25f31e219d34
Author: Hongzhi Gao <76...@qq.com>
AuthorDate: Tue Oct 28 09:41:30 2025 +0800
New java interfaces (#602)
* add table model insert record
* add tag filter eq
* Provides complete implementation of all TagFilter operations
* implement new java writing interface for tree model
* implement v4 tree-model read interface
* Adjusted the sequence of measurements
* Add licenses
* Add v4 interface example
* Add v4 PerformanceTest
* fix TableResultSet.Iterator unittest
* add license
* spotless:apply
* fix MeasurementSchemaBuilder getCompressor()
* removed unused IDeviceID.toString()
---------
Co-authored-by: Tian Jiang <jt...@163.com>
---
.../main/java/org/apache/tsfile/utils/Binary.java | 3 +
.../org/apache/tsfile/v4/TagFilterExample.java | 206 +++++++++
.../apache/tsfile/v4/TsFileTreeReaderExample.java | 181 ++++++++
.../src/main/codegen/dataModel/AllFilter.tdd | 5 +
.../codegen/templates/FilterOperatorsTemplate.ftl | 128 +++---
.../src/main/codegen/templates/FilterTemplate.ftl | 7 +
.../tsfile/file/metadata/ColumnSchemaBuilder.java | 15 +
.../tsfile/file/metadata/StringArrayDeviceID.java | 10 +-
.../apache/tsfile/file/metadata/TableSchema.java | 9 +
.../java/org/apache/tsfile/read/TsFileReader.java | 23 +
.../apache/tsfile/read/TsFileSequenceReader.java | 14 +
.../tsfile/read/controller/DeviceMetaIterator.java | 12 +-
.../tsfile/read/controller/IMetadataQuerier.java | 4 +-
.../read/controller/MetadataQuerierByFileImpl.java | 6 +-
.../apache/tsfile/read/filter/basic/Filter.java | 2 +
.../tsfile/read/filter/basic/TimeFilter.java | 5 +
.../tsfile/read/filter/basic/ValueFilter.java | 7 +-
.../read/filter/factory/TagFilterBuilder.java | 131 ++++++
.../apache/tsfile/read/filter/operator/And.java | 5 +
.../apache/tsfile/read/filter/operator/Not.java | 5 +
.../org/apache/tsfile/read/filter/operator/Or.java | 5 +
.../tsfile/read/query/dataset/ResultSet.java | 7 +-
.../tsfile/read/query/dataset/TableResultSet.java | 100 ++++-
.../tsfile/read/query/dataset/TreeResultSet.java | 123 +++++-
.../read/query/executor/TableQueryExecutor.java | 7 +-
.../query/executor/task/DeviceTaskIterator.java | 6 +-
.../tsfile/read/v4/DeviceTableModelReader.java | 12 +-
.../org/apache/tsfile/read/v4/ITsFileReader.java | 6 +
.../{ITsFileReader.java => ITsFileTreeReader.java} | 30 +-
.../apache/tsfile/read/v4/TsFileTreeReader.java | 127 ++++++
.../v4/TsFileTreeReaderBuilder.java} | 29 +-
.../org/apache/tsfile/utils/ReadWriteIOUtils.java | 23 +
.../java/org/apache/tsfile/write/TsFileWriter.java | 19 +
.../write/chunk/AlignedChunkGroupWriterImpl.java | 3 +
.../org/apache/tsfile/write/record/TSRecord.java | 45 +-
.../tsfile/write/record/datapoint/DataPoint.java | 11 +
.../write/schema/MeasurementSchemaBuilder.java | 135 ++++++
.../tsfile/write/v4/DeviceTableModelWriter.java | 40 ++
.../org/apache/tsfile/write/v4/ITsFileWriter.java | 4 +
.../apache/tsfile/write/v4/TsFileTreeWriter.java | 129 ++++++
.../tsfile/write/v4/TsFileTreeWriterBuilder.java | 68 +++
.../apache/tsfile/read/TsFileTreeReaderTest.java | 190 +++++++++
.../apache/tsfile/read/query/ResultSetTest.java | 369 ++++++++++++++++
.../apache/tsfile/tableview/PerformanceTest.java | 468 +++++++++++++--------
.../apache/tsfile/write/TsFileWriteApiTest.java | 30 ++
45 files changed, 2484 insertions(+), 280 deletions(-)
diff --git a/java/common/src/main/java/org/apache/tsfile/utils/Binary.java b/java/common/src/main/java/org/apache/tsfile/utils/Binary.java
index f65d19c3..07337174 100644
--- a/java/common/src/main/java/org/apache/tsfile/utils/Binary.java
+++ b/java/common/src/main/java/org/apache/tsfile/utils/Binary.java
@@ -109,6 +109,9 @@ public class Binary implements Comparable<Binary>, Serializable, Accountable {
@Override
public String toString() {
+ if (values == null) {
+ return null;
+ }
// use UTF_8 by default since toString do not provide parameter
return getStringValue(StandardCharsets.UTF_8);
}
diff --git a/java/examples/src/main/java/org/apache/tsfile/v4/TagFilterExample.java b/java/examples/src/main/java/org/apache/tsfile/v4/TagFilterExample.java
new file mode 100644
index 00000000..d78f873e
--- /dev/null
+++ b/java/examples/src/main/java/org/apache/tsfile/v4/TagFilterExample.java
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tsfile.v4;
+
+import org.apache.tsfile.enums.ColumnCategory;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.ColumnSchemaBuilder;
+import org.apache.tsfile.file.metadata.TableSchema;
+import org.apache.tsfile.read.filter.basic.Filter;
+import org.apache.tsfile.read.filter.factory.TagFilterBuilder;
+import org.apache.tsfile.read.query.dataset.ResultSet;
+import org.apache.tsfile.read.v4.ITsFileReader;
+import org.apache.tsfile.read.v4.TsFileReaderBuilder;
+import org.apache.tsfile.write.record.Tablet;
+import org.apache.tsfile.write.v4.ITsFileWriter;
+import org.apache.tsfile.write.v4.TsFileWriterBuilder;
+
+import java.io.File;
+import java.nio.file.Files;
+import java.util.Arrays;
+
+public class TagFilterExample {
+
+ public static void main(String[] args) throws Exception {
+ String path = "tag_filter_example.tsfile";
+ File f = new File(path);
+ if (f.exists()) {
+ Files.delete(f.toPath());
+ }
+
+ // Define table schema
+ TableSchema schema =
+ new TableSchema(
+ "t1",
+ Arrays.asList(
+ new ColumnSchemaBuilder()
+ .name("id1")
+ .dataType(TSDataType.STRING)
+ .category(ColumnCategory.TAG)
+ .build(),
+ new ColumnSchemaBuilder()
+ .name("id2")
+ .dataType(TSDataType.STRING)
+ .category(ColumnCategory.TAG)
+ .build(),
+ new ColumnSchemaBuilder()
+ .name("s1")
+ .dataType(TSDataType.BOOLEAN)
+ .category(ColumnCategory.FIELD)
+ .build(),
+ new ColumnSchemaBuilder()
+ .name("s2")
+ .dataType(TSDataType.BOOLEAN)
+ .category(ColumnCategory.FIELD)
+ .build()));
+
+ // Prepare tablet for writing data
+ Tablet tablet =
+ new Tablet(
+ Arrays.asList("id1", "id2", "s1", "s2"),
+ Arrays.asList(
+ TSDataType.STRING, TSDataType.STRING, TSDataType.BOOLEAN, TSDataType.BOOLEAN));
+
+ // Row 0
+ tablet.addTimestamp(0, 0);
+ tablet.addValue(0, "id1", "id_01");
+ tablet.addValue(0, "id2", "name_01");
+ tablet.addValue(0, "s1", true);
+ tablet.addValue(0, "s2", false);
+
+ // Row 1
+ tablet.addTimestamp(1, 1);
+ tablet.addValue(1, "id1", "id_02");
+ tablet.addValue(1, "id2", "name_02");
+ tablet.addValue(1, "s1", false);
+ tablet.addValue(1, "s2", true);
+
+ // Row 2
+ tablet.addTimestamp(2, 2);
+ tablet.addValue(2, "id1", "id_03");
+ tablet.addValue(2, "id2", "name_03");
+ tablet.addValue(2, "s1", true);
+ tablet.addValue(2, "s2", true);
+
+ // Row 3
+ tablet.addTimestamp(3, 3);
+ tablet.addValue(3, "id1", "id_04");
+ tablet.addValue(3, "id2", "name_04");
+ tablet.addValue(3, "s1", false);
+ tablet.addValue(3, "s2", false);
+
+ // Write tablet to TsFile
+ try (ITsFileWriter writer = new TsFileWriterBuilder().file(f).tableSchema(schema).build()) {
+ writer.write(tablet);
+ }
+
+ // ---------------- TagFilter Examples ----------------
+ TagFilterBuilder filterBuilder = new TagFilterBuilder(schema);
+
+ // eq
+ queryWithFilter(f, schema, filterBuilder.eq("id1", "id_02"), "eq(id1 = id_02)");
+
+ // neq
+ queryWithFilter(f, schema, filterBuilder.neq("id1", "id_02"), "neq(id1 != id_02)");
+
+ // lt
+ queryWithFilter(f, schema, filterBuilder.lt("id1", "id_03"), "lt(id1 < id_03)");
+
+ // lteq
+ queryWithFilter(f, schema, filterBuilder.lteq("id1", "id_03"), "lteq(id1 <= id_03)");
+
+ // gt
+ queryWithFilter(f, schema, filterBuilder.gt("id1", "id_02"), "gt(id1 > id_02)");
+
+ // gteq
+ queryWithFilter(f, schema, filterBuilder.gteq("id1", "id_02"), "gteq(id1 >= id_02)");
+
+ // betweenAnd
+ queryWithFilter(
+ f,
+ schema,
+ filterBuilder.betweenAnd("id1", "id_02", "id_03"),
+ "betweenAnd(id1 between id_02 and id_03)");
+
+ // notBetweenAnd
+ queryWithFilter(
+ f,
+ schema,
+ filterBuilder.notBetweenAnd("id1", "id_02", "id_03"),
+ "notBetweenAnd(id1 not between id_02 and id_03)");
+
+ // and
+ queryWithFilter(
+ f,
+ schema,
+ filterBuilder.and(filterBuilder.gteq("id1", "id_02"), filterBuilder.lteq("id1", "id_03")),
+ "and(gteq(id1,id_02), lteq(id1,id_03))");
+
+ // or
+ queryWithFilter(
+ f,
+ schema,
+ filterBuilder.or(filterBuilder.eq("id1", "id_01"), filterBuilder.eq("id1", "id_04")),
+ "or(eq(id1,id_01), eq(id1,id_04))");
+
+ // not
+ queryWithFilter(
+ f, schema, filterBuilder.not(filterBuilder.eq("id1", "id_02")), "not(eq(id1,id_02))");
+
+ // regExp
+ queryWithFilter(
+ f, schema, filterBuilder.regExp("id1", "id_0[23]"), "regExp(id1 matches id_0[23])");
+
+ // notRegExp
+ queryWithFilter(
+ f,
+ schema,
+ filterBuilder.notRegExp("id1", "id_0[23]"),
+ "notRegExp(id1 not matches id_0[23])");
+
+ // like
+ queryWithFilter(f, schema, filterBuilder.like("id1", "id_0_"), "like(id1 like id_0_)");
+
+ // notLike
+ queryWithFilter(
+ f, schema, filterBuilder.notLike("id1", "id_0_"), "notLike(id1 not like id_0_)");
+ new File(path).delete();
+ }
+
+ /** Helper method to execute a query with a given filter and print results. */
+ private static void queryWithFilter(File f, TableSchema schema, Object filter, String desc)
+ throws Exception {
+ try (ITsFileReader reader = new TsFileReaderBuilder().file(f).build();
+ ResultSet rs =
+ reader.query("t1", Arrays.asList("id1", "id2", "s1", "s2"), 0, 3, (Filter) filter)) {
+
+ System.out.println("=== Query with TagFilter " + desc + " ===");
+ while (rs.next()) {
+ long time = rs.getLong("Time");
+ String id1 = rs.getString("id1");
+ String id2 = rs.getString("id2");
+ Boolean s1 = rs.isNull("s1") ? null : rs.getBoolean("s1");
+ Boolean s2 = rs.isNull("s2") ? null : rs.getBoolean("s2");
+
+ System.out.printf("time=%d, id1=%s, id2=%s, s1=%s, s2=%s%n", time, id1, id2, s1, s2);
+ }
+ }
+ }
+}
diff --git a/java/examples/src/main/java/org/apache/tsfile/v4/TsFileTreeReaderExample.java b/java/examples/src/main/java/org/apache/tsfile/v4/TsFileTreeReaderExample.java
new file mode 100644
index 00000000..a317a22e
--- /dev/null
+++ b/java/examples/src/main/java/org/apache/tsfile/v4/TsFileTreeReaderExample.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tsfile.v4;
+
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.exception.write.WriteProcessException;
+import org.apache.tsfile.read.query.dataset.ResultSet;
+import org.apache.tsfile.read.query.dataset.ResultSetMetadata;
+import org.apache.tsfile.read.v4.ITsFileTreeReader;
+import org.apache.tsfile.read.v4.TsFileTreeReaderBuilder;
+import org.apache.tsfile.write.record.TSRecord;
+import org.apache.tsfile.write.record.Tablet;
+import org.apache.tsfile.write.schema.IMeasurementSchema;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+import org.apache.tsfile.write.v4.TsFileTreeWriter;
+import org.apache.tsfile.write.v4.TsFileTreeWriterBuilder;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * Example for writing and reading TsFile in tree model.
+ *
+ * <p>Demonstrates: - Defining schemas for multiple devices - Writing data using both Tablet and
+ * TSRecord - Reading back with TsFileTreeReader - Retrieving device IDs and schema info - Using
+ * ResultSet and ResultSetMetadata
+ */
+public class TsFileTreeReaderExample {
+
+ private static final String FILE_PATH = "tree_model_example.tsfile";
+
+ public static void main(String[] args) throws IOException {
+ File file = new File(FILE_PATH);
+ if (file.exists()) {
+ file.delete();
+ }
+
+ // Step 1. Generate TsFile with tree writer
+ generateTestTsFile(file);
+
+ List<String> deviceIds = Arrays.asList("d1", "d2");
+ List<String> measurements = Arrays.asList("s1", "s2");
+
+ // Step 2. Build tree reader
+ try (ITsFileTreeReader treeReader = new TsFileTreeReaderBuilder().file(file).build()) {
+
+ // === Basic query by devices and measurements ===
+
+ try (ResultSet resultSet =
+ treeReader.query(deviceIds, measurements, Long.MIN_VALUE, Long.MAX_VALUE)) {
+
+ System.out.println("=== Query results ===");
+ while (resultSet.next()) {
+ long time = resultSet.getLong(1);
+ Integer d1s1 = resultSet.isNull("d1.s1") ? null : resultSet.getInt("d1.s1");
+ Long d1s2 = resultSet.isNull("d1.s2") ? null : resultSet.getLong("d1.s2");
+ Integer d2s1 = resultSet.isNull("d2.s1") ? null : resultSet.getInt("d2.s1");
+
+ System.out.printf("time=%d, d1.s1=%s, d1.s2=%s, d2.s1=%s%n", time, d1s1, d1s2, d2s1);
+ }
+ }
+ // === Retrieve all device IDs ===
+ List<String> allDeviceIds = treeReader.getAllDeviceIds();
+ System.out.println("=== All device IDs === " + allDeviceIds);
+
+ // === Retrieve schema of device d1 ===
+ List<MeasurementSchema> schemas = treeReader.getDeviceSchema("d1");
+ List<String> measurementNames =
+ schemas.stream().map(IMeasurementSchema::getMeasurementName).collect(Collectors.toList());
+ System.out.println("=== Schema of d1 === " + measurementNames);
+
+ // === ResultSet metadata ===
+ try (ResultSet resultSet =
+ treeReader.query(deviceIds, measurements, Long.MIN_VALUE, Long.MAX_VALUE)) {
+ ResultSetMetadata metadata = resultSet.getMetadata();
+ System.out.println("=== Metadata columns ===");
+ for (int i = 1; i <= 4; i++) {
+ System.out.printf("Column %d: %s%n", i, metadata.getColumnName(i));
+ }
+ }
+ }
+
+ // === Iterator-based query example ===
+ try (ITsFileTreeReader treeReader = new TsFileTreeReaderBuilder().file(file).build()) {
+
+ long startTime = Long.MIN_VALUE;
+ long endTime = Long.MAX_VALUE;
+
+ try (ResultSet resultSet = treeReader.query(deviceIds, measurements, startTime, endTime)) {
+ System.out.println("=== Iterator-based query results ===");
+ Iterator<TSRecord> recordIterator = resultSet.recordIterator();
+ int countVal = 0;
+
+ while (recordIterator.hasNext()) {
+ TSRecord record = recordIterator.next();
+ System.out.println(
+ "Device: " + record.deviceId.toString() + ", Timestamp: " + record.time);
+
+ if (record.deviceId.toString().equals("d1")) {
+ System.out.println(
+ " Measurement at index 1: "
+ + record.dataPointList.get(1).getMeasurementId()
+ + " = "
+ + record.dataPointList.get(1).getValue());
+ System.out.println(
+ " Measurement at index 2: "
+ + record.dataPointList.get(2).getMeasurementId()
+ + " = "
+ + record.dataPointList.get(2).getValue());
+ } else if (record.deviceId.toString().equals("d2")) {
+ System.out.println(
+ " Measurement at index 1: "
+ + record.dataPointList.get(1).getMeasurementId()
+ + " = "
+ + record.dataPointList.get(1).getValue());
+ System.out.println(" Measurement at index 2 is null");
+ } else {
+ System.out.println(" Unexpected deviceId: " + record.deviceId.toString());
+ }
+
+ countVal++;
+ }
+
+ System.out.println("Total records iterated: " + countVal);
+ }
+ }
+ new File(FILE_PATH).delete();
+ }
+
+ private static void generateTestTsFile(File file) throws IOException {
+ try (TsFileTreeWriter writer = new TsFileTreeWriterBuilder().file(file).build()) {
+ // Define schemas
+ IMeasurementSchema schema1 = new MeasurementSchema("s1", TSDataType.INT32);
+ IMeasurementSchema schema2 = new MeasurementSchema("s2", TSDataType.INT64);
+
+ // Register schemas for devices
+ writer.registerTimeseries("d1", schema1);
+ writer.registerTimeseries("d1", schema2);
+ writer.registerTimeseries("d2", schema1);
+
+ // Write data for device d1 with Tablet
+ Tablet d1Tablet = new Tablet("d1", Arrays.asList(schema1, schema2));
+ for (int i = 0; i < 5; i++) {
+ d1Tablet.addTimestamp(i, i);
+ d1Tablet.addValue(i, 0, i); // s1
+ d1Tablet.addValue(i, 1, (long) i); // s2
+ }
+ writer.write(d1Tablet);
+
+ // Write data for device d2 with TSRecord
+ for (int i = 0; i < 5; i++) {
+ TSRecord record = new TSRecord("d2", i);
+ record.addPoint("s1", i);
+ writer.write(record);
+ }
+ } catch (WriteProcessException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git a/java/tsfile/src/main/codegen/dataModel/AllFilter.tdd b/java/tsfile/src/main/codegen/dataModel/AllFilter.tdd
index cb5815bc..d8dcd64e 100644
--- a/java/tsfile/src/main/codegen/dataModel/AllFilter.tdd
+++ b/java/tsfile/src/main/codegen/dataModel/AllFilter.tdd
@@ -52,6 +52,11 @@
"dataType": "Binary",
"javaBoxName": "String",
"classSerializeName": "STRING"
+ },
+ {
+ "dataType": "String",
+ "javaBoxName": "Tag",
+ "classSerializeName": "TAG"
}
]
}
\ No newline at end of file
diff --git a/java/tsfile/src/main/codegen/templates/FilterOperatorsTemplate.ftl b/java/tsfile/src/main/codegen/templates/FilterOperatorsTemplate.ftl
index 7e39b361..83538cf1 100644
--- a/java/tsfile/src/main/codegen/templates/FilterOperatorsTemplate.ftl
+++ b/java/tsfile/src/main/codegen/templates/FilterOperatorsTemplate.ftl
@@ -2,6 +2,12 @@
<#list filters as filter>
<#assign className = "${filter.javaBoxName}FilterOperators">
<#assign filterName = "${filter.javaBoxName}Filter">
+ <#if filter.javaBoxName == "Tag">
+ <#assign javaClassName = "String">
+ <#else>
+ <#assign javaClassName = "${filter.javaBoxName}">
+ </#if>
+
<@pp.changeOutputFile name="/org/apache/tsfile/read/filter/operator/${className}.java" />
/*
* Licensed to the Apache Software Foundation (ASF) under one
@@ -74,7 +80,7 @@ public final class ${className} {
protected ValueColumnCompareFilter(int measurementIndex, ${filter.dataType} constant) {
super(measurementIndex);
- <#if filter.dataType == "Binary">
+ <#if filter.dataType == "Binary" || filter.dataType == "String">
this.constant = Objects.requireNonNull(constant, CONSTANT_CANNOT_BE_NULL_MSG);
<#else>
this.constant = constant;
@@ -84,7 +90,7 @@ public final class ${className} {
@SuppressWarnings("unchecked")
protected ValueColumnCompareFilter(ByteBuffer buffer) {
super(buffer);
- <#if filter.dataType == "Binary">
+ <#if filter.dataType == "Binary" || filter.dataType == "String">
this.constant = Objects.requireNonNull(ReadWriteIOUtils.read${filter.dataType?cap_first}(buffer), CONSTANT_CANNOT_BE_NULL_MSG);
<#else>
this.constant = ReadWriteIOUtils.read${filter.dataType?cap_first}(buffer);
@@ -109,7 +115,7 @@ public final class ${className} {
return false;
}
ValueColumnCompareFilter that = (ValueColumnCompareFilter) o;
- <#if filter.dataType == "Binary">
+ <#if filter.dataType == "Binary" || filter.dataType == "String">
return Objects.equals(constant,that.constant);
<#else>
return constant == that.constant;
@@ -140,7 +146,7 @@ public final class ${className} {
@Override
public boolean valueSatisfy(Object value){
- <#if filter.dataType == "boolean" || filter.javaBoxName == "String">
+ <#if filter.dataType == "boolean" || filter.javaBoxName == "String" || filter.javaBoxName == "Tag">
return valueSatisfy((${filter.dataType}) value);
<#elseif filter.dataType == "Binary">
if(value instanceof Binary){
@@ -156,7 +162,7 @@ public final class ${className} {
@Override
public boolean valueSatisfy(${filter.dataType} value) {
- <#if filter.dataType == "Binary">
+ <#if filter.dataType == "Binary" || filter.dataType == "String">
return constant.equals(value);
<#else>
return constant == value;
@@ -166,8 +172,8 @@ public final class ${className} {
@Override
@SuppressWarnings("unchecked")
public boolean canSkip(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean">
- <#if filter.javaBoxName == "String">
+ <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String">
+ <#if filter.javaBoxName == "String" || filter.javaBoxName == "Tag">
if(statistics.isEmpty()){
return false;
}
@@ -205,7 +211,7 @@ public final class ${className} {
@Override
@SuppressWarnings("unchecked")
public boolean allSatisfy(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean">
+ <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String">
<#if filter.javaBoxName == "String">
if(statistics.isEmpty()){
return false;
@@ -264,7 +270,7 @@ public final class ${className} {
@Override
public boolean valueSatisfy(Object value){
- <#if filter.dataType == "boolean" || filter.javaBoxName == "String">
+ <#if filter.dataType == "boolean" || filter.javaBoxName == "String" || filter.javaBoxName == "Tag">
return valueSatisfy((${filter.dataType}) value);
<#elseif filter.dataType == "Binary">
if(value instanceof Binary){
@@ -279,7 +285,7 @@ public final class ${className} {
@Override
public boolean valueSatisfy(${filter.dataType} value) {
- <#if filter.dataType == "Binary">
+ <#if filter.dataType == "Binary" || filter.dataType == "String">
return !constant.equals(value);
<#else>
return constant != value;
@@ -289,7 +295,7 @@ public final class ${className} {
@Override
@SuppressWarnings("unchecked")
public boolean canSkip(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean">
+ <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String">
<#if filter.javaBoxName == "String">
if(statistics.isEmpty()){
return false;
@@ -329,7 +335,7 @@ public final class ${className} {
@Override
@SuppressWarnings("unchecked")
public boolean allSatisfy(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean">
+ <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String">
<#if filter.javaBoxName == "String">
if(statistics.isEmpty()){
return false;
@@ -388,7 +394,7 @@ public final class ${className} {
@Override
@SuppressWarnings("unchecked")
public boolean valueSatisfy(Object value){
- <#if filter.dataType == "boolean" || filter.javaBoxName == "String">
+ <#if filter.dataType == "boolean" || filter.javaBoxName == "String" || filter.javaBoxName == "Tag">
return valueSatisfy((${filter.dataType}) value);
<#elseif filter.dataType == "Binary">
if(value instanceof Binary){
@@ -405,7 +411,7 @@ public final class ${className} {
public boolean valueSatisfy(${filter.dataType} value) {
<#if filter.dataType == "boolean">
return Boolean.compare(constant,value) > 0;
- <#elseif filter.dataType == "Binary">
+ <#elseif filter.dataType == "Binary" || filter.dataType == "String">
return constant.compareTo(value) > 0;
<#else>
return constant > value;
@@ -415,7 +421,7 @@ public final class ${className} {
@Override
@SuppressWarnings("unchecked")
public boolean canSkip(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean">
+ <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String">
<#if filter.javaBoxName == "String">
if(statistics.isEmpty()){
return false;
@@ -451,7 +457,7 @@ public final class ${className} {
@Override
@SuppressWarnings("unchecked")
public boolean allSatisfy(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean">
+ <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String">
<#if filter.javaBoxName == "String">
if(statistics.isEmpty()){
return false;
@@ -506,7 +512,7 @@ public final class ${className} {
@Override
@SuppressWarnings("unchecked")
public boolean valueSatisfy(Object value){
- <#if filter.dataType == "boolean" || filter.javaBoxName == "String">
+ <#if filter.dataType == "boolean" || filter.javaBoxName == "String" || filter.javaBoxName == "Tag">
return valueSatisfy((${filter.dataType}) value);
<#elseif filter.dataType == "Binary">
if(value instanceof Binary){
@@ -523,7 +529,7 @@ public final class ${className} {
public boolean valueSatisfy(${filter.dataType} value) {
<#if filter.dataType == "boolean">
return Boolean.compare(constant,value) >= 0;
- <#elseif filter.dataType == "Binary">
+ <#elseif filter.dataType == "Binary" || filter.dataType == "String">
return constant.compareTo(value) >= 0;
<#else>
return constant >= value;
@@ -533,7 +539,7 @@ public final class ${className} {
@Override
@SuppressWarnings("unchecked")
public boolean canSkip(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean">
+ <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String">
<#if filter.javaBoxName == "String">
if(statistics.isEmpty()){
return false;
@@ -569,7 +575,7 @@ public final class ${className} {
@Override
@SuppressWarnings("unchecked")
public boolean allSatisfy(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean">
+ <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String">
<#if filter.javaBoxName == "String">
if(statistics.isEmpty()){
return false;
@@ -624,7 +630,7 @@ public final class ${className} {
@Override
@SuppressWarnings("unchecked")
public boolean valueSatisfy(Object value){
- <#if filter.dataType == "boolean" || filter.javaBoxName == "String">
+ <#if filter.dataType == "boolean" || filter.javaBoxName == "String" || filter.javaBoxName == "Tag">
return valueSatisfy((${filter.dataType}) value);
<#elseif filter.dataType == "Binary">
if(value instanceof Binary){
@@ -641,7 +647,7 @@ public final class ${className} {
public boolean valueSatisfy(${filter.dataType} value) {
<#if filter.dataType == "boolean">
return Boolean.compare(constant,value) < 0;
- <#elseif filter.dataType == "Binary">
+ <#elseif filter.dataType == "Binary" || filter.dataType == "String">
return constant.compareTo(value) < 0;
<#else>
return constant < value;
@@ -651,7 +657,7 @@ public final class ${className} {
@Override
@SuppressWarnings("unchecked")
public boolean canSkip(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean">
+ <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String">
<#if filter.javaBoxName == "String">
if(statistics.isEmpty()){
return false;
@@ -687,7 +693,7 @@ public final class ${className} {
@Override
@SuppressWarnings("unchecked")
public boolean allSatisfy(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean">
+ <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String">
<#if filter.javaBoxName == "String">
if(statistics.isEmpty()){
return false;
@@ -742,7 +748,7 @@ public final class ${className} {
@Override
@SuppressWarnings("unchecked")
public boolean valueSatisfy(Object value){
- <#if filter.dataType == "boolean" || filter.javaBoxName == "String">
+ <#if filter.dataType == "boolean" || filter.javaBoxName == "String" || filter.javaBoxName == "Tag">
return valueSatisfy((${filter.dataType}) value);
<#elseif filter.dataType == "Binary">
if(value instanceof Binary){
@@ -759,7 +765,7 @@ public final class ${className} {
public boolean valueSatisfy(${filter.dataType} value) {
<#if filter.dataType == "boolean">
return Boolean.compare(constant,value) <= 0;
- <#elseif filter.dataType == "Binary">
+ <#elseif filter.dataType == "Binary" || filter.dataType == "String">
return constant.compareTo(value) <= 0;
<#else>
return constant <= value;
@@ -769,7 +775,7 @@ public final class ${className} {
@Override
@SuppressWarnings("unchecked")
public boolean canSkip(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean">
+ <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String">
<#if filter.javaBoxName == "String">
if(statistics.isEmpty()){
return false;
@@ -805,7 +811,7 @@ public final class ${className} {
@Override
@SuppressWarnings("unchecked")
public boolean allSatisfy(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean">
+ <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String">
<#if filter.javaBoxName == "String">
if(statistics.isEmpty()){
return false;
@@ -855,7 +861,7 @@ public final class ${className} {
protected ValueColumnRangeFilter(int measurementIndex, ${filter.dataType} min, ${filter.dataType} max) {
super(measurementIndex);
- <#if filter.dataType == "Binary">
+ <#if filter.dataType == "Binary" || filter.dataType == "String">
this.min = Objects.requireNonNull(min,"min cannot be null");
this.max = Objects.requireNonNull(max,"max cannot be null");
<#else>
@@ -867,7 +873,7 @@ public final class ${className} {
@SuppressWarnings("unchecked")
protected ValueColumnRangeFilter(ByteBuffer buffer) {
super(buffer);
- <#if filter.dataType == "Binary">
+ <#if filter.dataType == "Binary" || filter.dataType == "String">
this.min = Objects.requireNonNull(ReadWriteIOUtils.read${filter.dataType?cap_first}(buffer),"min cannot be null");
this.max = Objects.requireNonNull(ReadWriteIOUtils.read${filter.dataType?cap_first}(buffer),"max cannot be null");
<#else>
@@ -895,7 +901,7 @@ public final class ${className} {
return false;
}
ValueColumnRangeFilter that = (ValueColumnRangeFilter) o;
- <#if filter.dataType == "Binary">
+ <#if filter.dataType == "Binary" || filter.dataType == "String">
return min.equals(that.min) && max.equals(that.max);
<#else>
return min == that.min && max == that.max;
@@ -928,7 +934,7 @@ public final class ${className} {
@Override
@SuppressWarnings("unchecked")
public boolean valueSatisfy(Object value){
- <#if filter.dataType == "boolean" || filter.javaBoxName == "String">
+ <#if filter.dataType == "boolean" || filter.javaBoxName == "String" || filter.javaBoxName == "Tag">
return valueSatisfy((${filter.dataType}) value);
<#elseif filter.dataType == "Binary">
if(value instanceof Binary){
@@ -946,7 +952,7 @@ public final class ${className} {
<#if filter.dataType == "boolean">
return Boolean.compare(min,value) <= 0
&& Boolean.compare(max,value) >= 0;
- <#elseif filter.dataType == "Binary">
+ <#elseif filter.dataType == "Binary" || filter.dataType == "String">
return min.compareTo(value) <= 0
&& max.compareTo(value) >= 0;
<#else>
@@ -957,7 +963,7 @@ public final class ${className} {
@Override
@SuppressWarnings("unchecked")
public boolean canSkip(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean">
+ <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String">
<#if filter.javaBoxName == "String">
if(statistics.isEmpty()){
return false;
@@ -996,7 +1002,7 @@ public final class ${className} {
@Override
@SuppressWarnings("unchecked")
public boolean allSatisfy(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean">
+ <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String">
<#if filter.javaBoxName == "String">
if(statistics.isEmpty()){
return false;
@@ -1055,7 +1061,7 @@ public final class ${className} {
@Override
@SuppressWarnings("unchecked")
public boolean valueSatisfy(Object value){
- <#if filter.dataType == "boolean" || filter.javaBoxName == "String">
+ <#if filter.dataType == "boolean" || filter.javaBoxName == "String" || filter.javaBoxName == "Tag">
return valueSatisfy((${filter.dataType}) value);
<#elseif filter.dataType == "Binary">
if(value instanceof Binary){
@@ -1073,7 +1079,7 @@ public final class ${className} {
<#if filter.dataType == "boolean">
return Boolean.compare(min,value) > 0
|| Boolean.compare(max,value) < 0;
- <#elseif filter.dataType == "Binary">
+ <#elseif filter.dataType == "Binary" || filter.dataType == "String">
return min.compareTo(value) > 0 || max.compareTo(value) < 0;
<#else>
return min > value || max < value;
@@ -1083,7 +1089,7 @@ public final class ${className} {
@Override
@SuppressWarnings("unchecked")
public boolean canSkip(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean">
+ <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String">
<#if filter.javaBoxName == "String">
if(statistics.isEmpty()){
return false;
@@ -1122,7 +1128,7 @@ public final class ${className} {
@Override
@SuppressWarnings("unchecked")
public boolean allSatisfy(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean">
+ <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String">
<#if filter.javaBoxName == "String">
if(statistics.isEmpty()){
return false;
@@ -1174,7 +1180,7 @@ public final class ${className} {
<#if filter.javaBoxName == "String">
protected final Set<${filter.dataType}> candidates;
<#else>
- protected final Set<${filter.javaBoxName}> candidates;
+ protected final Set<${javaClassName}> candidates;
</#if>
protected final ${filter.dataType} candidatesMin;
@@ -1183,7 +1189,7 @@ public final class ${className} {
<#if filter.javaBoxName == "String">
protected ValueColumnSetFilter(int measurementIndex, Set<${filter.dataType}> candidates) {
<#else>
- protected ValueColumnSetFilter(int measurementIndex, Set<${filter.javaBoxName}> candidates) {
+ protected ValueColumnSetFilter(int measurementIndex, Set<${javaClassName}> candidates) {
</#if>
super(measurementIndex);
this.candidates = candidates;
@@ -1191,18 +1197,18 @@ public final class ${className} {
<#if filter.javaBoxName == "String">
Set<${filter.dataType}> filteredSet = candidates.stream().filter(Objects::nonNull).collect(Collectors.toSet());
<#else>
- Set<${filter.javaBoxName}> filteredSet = candidates.stream().filter(Objects::nonNull).collect(Collectors.toSet());
+ Set<${javaClassName}> filteredSet = candidates.stream().filter(Objects::nonNull).collect(Collectors.toSet());
</#if>
<#if filter.dataType == "boolean">
// BooleanStatistics is not available
this.candidatesMin = false;
this.candidatesMax = false;
- <#elseif filter.dataType == "Binary">
+ <#elseif filter.dataType == "Binary" || filter.dataType == "String">
this.candidatesMin = !filteredSet.isEmpty() ? Collections.min(filteredSet) : null;
this.candidatesMax = !filteredSet.isEmpty() ? Collections.max(filteredSet) : null;
<#else>
- this.candidatesMin = !filteredSet.isEmpty() ? Collections.min(filteredSet) : ${filter.javaBoxName}.MIN_VALUE;
- this.candidatesMax = !filteredSet.isEmpty() ? Collections.max(filteredSet) : ${filter.javaBoxName}.MAX_VALUE;
+ this.candidatesMin = !filteredSet.isEmpty() ? Collections.min(filteredSet) : ${javaClassName}.MIN_VALUE;
+ this.candidatesMax = !filteredSet.isEmpty() ? Collections.max(filteredSet) : ${javaClassName}.MAX_VALUE;
</#if>
}
@@ -1212,18 +1218,18 @@ public final class ${className} {
<#if filter.javaBoxName == "String">
this.candidates = ReadWriteIOUtils.read${filter.dataType}Set(buffer);
<#else>
- this.candidates = ReadWriteIOUtils.read${filter.javaBoxName}Set(buffer);
+ this.candidates = ReadWriteIOUtils.read${javaClassName}Set(buffer);
</#if>
<#if filter.dataType == "boolean">
// BooleanStatistics is not available
this.candidatesMin = false;
this.candidatesMax = false;
- <#elseif filter.dataType == "Binary">
+ <#elseif filter.dataType == "Binary" || filter.dataType == "String">
this.candidatesMin = !candidates.isEmpty() ? Collections.min(candidates) : null;
this.candidatesMax = !candidates.isEmpty() ? Collections.max(candidates) : null;
<#else>
- this.candidatesMin = !candidates.isEmpty() ? Collections.min(candidates) : ${filter.javaBoxName}.MAX_VALUE;
- this.candidatesMax = !candidates.isEmpty() ? Collections.max(candidates) : ${filter.javaBoxName}.MAX_VALUE;
+ this.candidatesMin = !candidates.isEmpty() ? Collections.min(candidates) : ${javaClassName}.MAX_VALUE;
+ this.candidatesMax = !candidates.isEmpty() ? Collections.max(candidates) : ${javaClassName}.MAX_VALUE;
</#if>
if(hasNull){
this.candidates.add(null);
@@ -1237,7 +1243,7 @@ public final class ${className} {
<#if filter.javaBoxName == "String">
ReadWriteIOUtils.write${filter.dataType}Set(candidates, outputStream);
<#else>
- ReadWriteIOUtils.write${filter.javaBoxName}Set(candidates, outputStream);
+ ReadWriteIOUtils.write${javaClassName}Set(candidates, outputStream);
</#if>
}
@@ -1275,7 +1281,7 @@ public final class ${className} {
super(measurementIndex, candidates);
}
<#else>
- public ValueIn(int measurementIndex, Set<${filter.javaBoxName}> candidates) {
+ public ValueIn(int measurementIndex, Set<${javaClassName}> candidates) {
super(measurementIndex, candidates);
}
</#if>
@@ -1286,7 +1292,7 @@ public final class ${className} {
@Override
public boolean valueSatisfy(Object value){
- <#if filter.dataType == "boolean" || filter.javaBoxName == "String">
+ <#if filter.dataType == "boolean" || filter.javaBoxName == "String" || filter.javaBoxName == "Tag">
return candidates.contains((${filter.dataType}) value);
<#elseif filter.dataType == "Binary">
if(value instanceof Binary){
@@ -1324,7 +1330,7 @@ public final class ${className} {
if (statistics.isPresent()) {
Statistics<? extends Serializable> stat = statistics.get();
- <#if filter.dataType == "Binary" && filter.javaBoxName == "String">
+ <#if (filter.dataType == "Binary" && filter.javaBoxName == "String") || (filter.dataType == "String" && filter.javaBoxName == "Tag")>
${filter.dataType} valuesMin = (${filter.dataType}) stat.getMinValue();
${filter.dataType} valuesMax = (${filter.dataType}) stat.getMaxValue();
<#else>
@@ -1337,7 +1343,7 @@ public final class ${className} {
} else {
if (!candidates.isEmpty()) {
// All values are less than min, or greater than max
- <#if filter.dataType == "Binary" && filter.javaBoxName == "String">
+ <#if (filter.dataType == "Binary" && filter.javaBoxName == "String") || (filter.dataType == "String" && filter.javaBoxName == "Tag")>
return candidatesMin.compareTo(valuesMax) > 0
|| candidatesMax.compareTo(valuesMin) < 0;
<#else>
@@ -1378,7 +1384,7 @@ public final class ${className} {
// All values are same
if (statistics.isPresent()) {
Statistics<? extends Serializable> stat = statistics.get();
- <#if filter.dataType == "Binary" && filter.javaBoxName == "String">
+ <#if (filter.dataType == "Binary" && filter.javaBoxName == "String") || (filter.dataType == "String" && filter.javaBoxName == "Tag")>
${filter.dataType} valuesMin = (${filter.dataType}) stat.getMinValue();
${filter.dataType} valuesMax = (${filter.dataType}) stat.getMaxValue();
<#else>
@@ -1429,7 +1435,7 @@ public final class ${className} {
super(measurementIndex, candidates);
}
<#else>
- public ValueNotIn(int measurementIndex, Set<${filter.javaBoxName}> candidates) {
+ public ValueNotIn(int measurementIndex, Set<${javaClassName}> candidates) {
super(measurementIndex, candidates);
}
</#if>
@@ -1440,7 +1446,7 @@ public final class ${className} {
@Override
public boolean valueSatisfy(Object value){
- <#if filter.dataType == "boolean" || filter.javaBoxName == "String">
+ <#if filter.dataType == "boolean" || filter.javaBoxName == "String" || filter.javaBoxName == "Tag">
return !candidates.contains((${filter.dataType}) value);
<#elseif filter.dataType == "Binary">
if(value instanceof Binary){
@@ -1547,7 +1553,7 @@ public final class ${className} {
@Override
public boolean valueSatisfy(${filter.dataType} value) {
- <#if filter.dataType == "Binary">
+ <#if filter.dataType == "Binary" || filter.dataType == "String">
return pattern.matcher(new MatcherInput(value.toString(), new AccessCount())).find();
<#else>
return pattern.matcher(new MatcherInput(String.valueOf(value), new AccessCount())).find();
@@ -1592,7 +1598,7 @@ public final class ${className} {
@Override
public boolean valueSatisfy(${filter.dataType} value) {
- <#if filter.dataType == "Binary">
+ <#if filter.dataType == "Binary" || filter.dataType == "String">
return !pattern.matcher(new MatcherInput(value.toString(), new AccessCount())).find();
<#else>
return !pattern.matcher(new MatcherInput(String.valueOf(value), new AccessCount())).find();
@@ -1697,7 +1703,7 @@ public final class ${className} {
@Override
public boolean valueSatisfy(${filter.dataType} value) {
- <#if filter.dataType == "Binary">
+ <#if filter.dataType == "Binary" || filter.dataType == "String">
return pattern.getMatcher().match(value.toString().getBytes());
<#else>
return pattern.getMatcher().match(String.valueOf(value).getBytes());
@@ -1742,7 +1748,7 @@ public final class ${className} {
@Override
public boolean valueSatisfy(${filter.dataType} value) {
- <#if filter.dataType == "Binary">
+ <#if filter.dataType == "Binary" || filter.dataType == "String">
return !pattern.getMatcher().match(value.toString().getBytes());
<#else>
return !pattern.getMatcher().match(String.valueOf(value).getBytes());
diff --git a/java/tsfile/src/main/codegen/templates/FilterTemplate.ftl b/java/tsfile/src/main/codegen/templates/FilterTemplate.ftl
index e1fd9fef..d0574842 100644
--- a/java/tsfile/src/main/codegen/templates/FilterTemplate.ftl
+++ b/java/tsfile/src/main/codegen/templates/FilterTemplate.ftl
@@ -66,6 +66,7 @@ public abstract class ${className} extends ValueFilter {
protected abstract boolean valueSatisfy(${filter.dataType} value);
+ <#if filter.javaBoxName != "Tag">
@Override
public boolean[] satisfyTsBlock(boolean[] selection, TsBlock tsBlock) {
Column valueColumn = tsBlock.getValueColumns()[measurementIndex];
@@ -83,6 +84,12 @@ public abstract class ${className} extends ValueFilter {
}
return satisfyInfo;
}
+ <#else >
+ @Override
+ public boolean[] satisfyTsBlock(boolean[] selection, TsBlock tsBlock) {
+ throw new IllegalArgumentException("TagFilter cannot be applied to TsBlock");
+ }
+ </#if>
@Override
public void serialize(DataOutputStream outputStream) throws IOException {
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/ColumnSchemaBuilder.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/ColumnSchemaBuilder.java
index 8b43c872..01896dc2 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/ColumnSchemaBuilder.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/ColumnSchemaBuilder.java
@@ -56,6 +56,21 @@ public class ColumnSchemaBuilder {
return this;
}
+ @TsFileApi
+ public String getColumnName() {
+ return columnName;
+ }
+
+ @TsFileApi
+ public TSDataType getDataType() {
+ return columnDataType;
+ }
+
+ @TsFileApi
+ public ColumnCategory getColumnCategory() {
+ return columnCategory;
+ }
+
private void validateParameters() {
if (columnName == null) {
throw new IllegalStateException("Column name must be set before building");
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/StringArrayDeviceID.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/StringArrayDeviceID.java
index 6f093add..895b2870 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/StringArrayDeviceID.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/StringArrayDeviceID.java
@@ -226,6 +226,11 @@ public class StringArrayDeviceID implements IDeviceID {
return !segments[0].startsWith(PATH_ROOT + PATH_SEPARATOR);
}
+ @Override
+ public String toString() {
+ return String.join(PATH_SEPARATOR, segments);
+ }
+
@Override
public String getTableName() {
return segments[0];
@@ -299,11 +304,6 @@ public class StringArrayDeviceID implements IDeviceID {
return cnt;
}
- @Override
- public String toString() {
- return String.join(".", segments);
- }
-
@Override
public boolean equals(Object o) {
if (this == o) {
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchema.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchema.java
index 81964473..70ddc690 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchema.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchema.java
@@ -53,6 +53,7 @@ public class TableSchema {
private Map<String, Integer> columnPosIndex;
// columnName -> pos in all id columns
private Map<String, Integer> idColumnOrder;
+ private int tagColumnCnt = -1;
public TableSchema(String tableName) {
this.tableName = tableName.toLowerCase();
@@ -331,4 +332,12 @@ public class TableSchema {
public int hashCode() {
return Objects.hash(tableName, measurementSchemas, columnCategories);
}
+
+ public int getTagColumnCnt() {
+ if (tagColumnCnt != -1) {
+ return tagColumnCnt;
+ }
+ tagColumnCnt = (int) columnCategories.stream().filter(c -> c == ColumnCategory.TAG).count();
+ return tagColumnCnt;
+ }
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileReader.java
index 15860726..f333afed 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileReader.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileReader.java
@@ -20,6 +20,8 @@
package org.apache.tsfile.read;
import org.apache.tsfile.annotations.TsFileApi;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.read.controller.CachedChunkLoaderImpl;
import org.apache.tsfile.read.controller.IChunkLoader;
import org.apache.tsfile.read.controller.IMetadataQuerier;
@@ -27,9 +29,14 @@ import org.apache.tsfile.read.controller.MetadataQuerierByFileImpl;
import org.apache.tsfile.read.expression.QueryExpression;
import org.apache.tsfile.read.query.dataset.QueryDataSet;
import org.apache.tsfile.read.query.executor.TsFileExecutor;
+import org.apache.tsfile.write.schema.MeasurementSchema;
import java.io.File;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
public class TsFileReader implements AutoCloseable {
@@ -61,6 +68,22 @@ public class TsFileReader implements AutoCloseable {
return tsfileExecutor.execute(queryExpression, partitionStartOffset, partitionEndOffset);
}
+ public List<String> getAllDeviceIds() throws IOException {
+ List<IDeviceID> deviceIDList = fileReader.getAllDevices();
+ List<String> deviceIds = new ArrayList<>(deviceIDList.size());
+ for (IDeviceID deviceID : deviceIDList) {
+ deviceIds.add(deviceID.toString());
+ }
+ return deviceIds;
+ }
+
+ public List<MeasurementSchema> getMeasurement(IDeviceID deviceID) throws IOException {
+ Map<String, TSDataType> measurementMap = fileReader.getMeasurement(deviceID);
+ return measurementMap.entrySet().stream()
+ .map(entry -> new MeasurementSchema(entry.getKey(), entry.getValue()))
+ .collect(Collectors.toList());
+ }
+
@Override
@TsFileApi
public void close() throws IOException {
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java
index 9c6d8ed4..7a572b9f 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java
@@ -2865,6 +2865,20 @@ public class TsFileSequenceReader implements AutoCloseable {
return result;
}
+ /**
+ * get measurements for deviceID
+ *
+ * @return measurementSchema
+ */
+ public Map<String, TSDataType> getMeasurement(IDeviceID deviceID) throws IOException {
+ Map<String, TSDataType> result = new HashMap<>();
+ Map<String, TimeseriesMetadata> timeseriesMetadataMap = readDeviceMetadata(deviceID);
+ for (TimeseriesMetadata timeseriesMetadata : timeseriesMetadataMap.values()) {
+ result.put(timeseriesMetadata.getMeasurementId(), timeseriesMetadata.getTsDataType());
+ }
+ return result;
+ }
+
/**
* get all types of measurements in this file
*
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/controller/DeviceMetaIterator.java b/java/tsfile/src/main/java/org/apache/tsfile/read/controller/DeviceMetaIterator.java
index 4445bf9b..74c99efa 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/controller/DeviceMetaIterator.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/controller/DeviceMetaIterator.java
@@ -24,7 +24,7 @@ import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.file.metadata.MetadataIndexNode;
import org.apache.tsfile.file.metadata.enums.MetadataIndexNodeType;
import org.apache.tsfile.read.TsFileSequenceReader;
-import org.apache.tsfile.read.expression.ExpressionTree;
+import org.apache.tsfile.read.filter.basic.Filter;
import org.apache.tsfile.utils.Pair;
import org.slf4j.Logger;
@@ -43,15 +43,15 @@ public class DeviceMetaIterator implements Iterator<Pair<IDeviceID, MetadataInde
private final TsFileSequenceReader tsFileSequenceReader;
private final Queue<MetadataIndexNode> metadataIndexNodes = new ArrayDeque<>();
private final Queue<Pair<IDeviceID, MetadataIndexNode>> resultCache = new ArrayDeque<>();
- private final ExpressionTree idFilter;
+ private final Filter tagFilter;
public DeviceMetaIterator(
TsFileSequenceReader tsFileSequenceReader,
MetadataIndexNode metadataIndexNode,
- ExpressionTree idFilter) {
+ Filter tagFilter) {
this.tsFileSequenceReader = tsFileSequenceReader;
this.metadataIndexNodes.add(metadataIndexNode);
- this.idFilter = idFilter;
+ this.tagFilter = tagFilter;
}
@Override
@@ -74,7 +74,8 @@ public class DeviceMetaIterator implements Iterator<Pair<IDeviceID, MetadataInde
for (int i = 0; i < leafChildren.size(); i++) {
IMetadataIndexEntry child = leafChildren.get(i);
final IDeviceID deviceID = ((DeviceMetadataIndexEntry) child).getDeviceID();
- if (idFilter != null && !idFilter.satisfy(deviceID)) {
+ // time is only a placeholder here
+ if (tagFilter != null && !tagFilter.satisfyRow(0, deviceID.getSegments())) {
continue;
}
@@ -114,6 +115,7 @@ public class DeviceMetaIterator implements Iterator<Pair<IDeviceID, MetadataInde
if (!resultCache.isEmpty()) {
return;
}
+ break;
case INTERNAL_DEVICE:
loadInternalNode(currentNode);
break;
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/controller/IMetadataQuerier.java b/java/tsfile/src/main/java/org/apache/tsfile/read/controller/IMetadataQuerier.java
index 3503e49e..3f74178d 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/controller/IMetadataQuerier.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/controller/IMetadataQuerier.java
@@ -28,7 +28,7 @@ import org.apache.tsfile.file.metadata.TableSchema;
import org.apache.tsfile.file.metadata.TsFileMetadata;
import org.apache.tsfile.read.common.Path;
import org.apache.tsfile.read.common.TimeRange;
-import org.apache.tsfile.read.expression.ExpressionTree;
+import org.apache.tsfile.read.filter.basic.Filter;
import org.apache.tsfile.utils.Pair;
import java.io.IOException;
@@ -87,5 +87,5 @@ public interface IMetadataQuerier {
void clear();
Iterator<Pair<IDeviceID, MetadataIndexNode>> deviceIterator(
- MetadataIndexNode root, ExpressionTree idFilter);
+ MetadataIndexNode root, Filter idFilter);
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/controller/MetadataQuerierByFileImpl.java b/java/tsfile/src/main/java/org/apache/tsfile/read/controller/MetadataQuerierByFileImpl.java
index ae7fb9ee..68e19530 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/controller/MetadataQuerierByFileImpl.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/controller/MetadataQuerierByFileImpl.java
@@ -34,7 +34,7 @@ import org.apache.tsfile.read.TsFileSequenceReader;
import org.apache.tsfile.read.TsFileSequenceReader.LocateStatus;
import org.apache.tsfile.read.common.Path;
import org.apache.tsfile.read.common.TimeRange;
-import org.apache.tsfile.read.expression.ExpressionTree;
+import org.apache.tsfile.read.filter.basic.Filter;
import org.apache.tsfile.utils.Pair;
import java.io.IOException;
@@ -306,7 +306,7 @@ public class MetadataQuerierByFileImpl implements IMetadataQuerier {
@Override
public Iterator<Pair<IDeviceID, MetadataIndexNode>> deviceIterator(
- MetadataIndexNode root, ExpressionTree idFilter) {
- return new DeviceMetaIterator(tsFileReader, root, idFilter);
+ MetadataIndexNode root, Filter tagFilter) {
+ return new DeviceMetaIterator(tsFileReader, root, tagFilter);
}
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/basic/Filter.java b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/basic/Filter.java
index 00801fd0..3bffa189 100755
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/basic/Filter.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/basic/Filter.java
@@ -79,6 +79,8 @@ public abstract class Filter {
public abstract boolean satisfyBinary(long time, Binary value);
+ public abstract boolean satisfyString(long time, String value);
+
/**
* To examine whether the row(with time and values) is satisfied with the filter.
*
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/basic/TimeFilter.java b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/basic/TimeFilter.java
index f1789b4b..1c37b236 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/basic/TimeFilter.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/basic/TimeFilter.java
@@ -74,6 +74,11 @@ public abstract class TimeFilter extends Filter {
return timeSatisfy(time);
}
+ @Override
+ public boolean satisfyString(long time, String value) {
+ return timeSatisfy(time);
+ }
+
@Override
public boolean satisfyRow(long time, Object[] values) {
// only use time to filter
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/basic/ValueFilter.java b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/basic/ValueFilter.java
index e1ae98bb..d715352f 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/basic/ValueFilter.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/basic/ValueFilter.java
@@ -84,6 +84,11 @@ public abstract class ValueFilter extends Filter {
throw new UnSupportedDataTypeException(getClass().getName());
}
+ @Override
+ public boolean satisfyString(long time, String value) {
+ throw new UnSupportedDataTypeException(getClass().getName());
+ }
+
@Override
public boolean satisfyBinary(long time, Binary value) {
throw new UnSupportedDataTypeException(getClass().getName());
@@ -91,7 +96,7 @@ public abstract class ValueFilter extends Filter {
@Override
public boolean satisfyRow(long time, Object[] values) {
- return satisfy(time, values[measurementIndex]);
+ return satisfy(time, measurementIndex < values.length ? values[measurementIndex] : null);
}
@Override
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/factory/TagFilterBuilder.java b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/factory/TagFilterBuilder.java
new file mode 100644
index 00000000..72339574
--- /dev/null
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/factory/TagFilterBuilder.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tsfile.read.filter.factory;
+
+import org.apache.tsfile.annotations.TsFileApi;
+import org.apache.tsfile.common.regexp.LikePattern;
+import org.apache.tsfile.file.metadata.TableSchema;
+import org.apache.tsfile.read.filter.basic.Filter;
+import org.apache.tsfile.read.filter.operator.And;
+import org.apache.tsfile.read.filter.operator.Not;
+import org.apache.tsfile.read.filter.operator.Or;
+import org.apache.tsfile.read.filter.operator.TagFilterOperators;
+import org.apache.tsfile.read.filter.operator.TagFilterOperators.ValueEq;
+import org.apache.tsfile.read.filter.operator.TagFilterOperators.ValueNotEq;
+
+import java.util.Optional;
+import java.util.regex.Pattern;
+
+public class TagFilterBuilder {
+ private final TableSchema tableSchema;
+
+ public TagFilterBuilder(TableSchema tableSchema) {
+ this.tableSchema = tableSchema;
+ }
+
+ private int getIdColumnIndex(String columnName) {
+ int idColumnOrder = tableSchema.findIdColumnOrder(columnName);
+ if (idColumnOrder == -1) {
+ throw new IllegalArgumentException("Column '" + columnName + "' is not a tag column");
+ }
+ return idColumnOrder + 1;
+ }
+
+ @TsFileApi
+ public Filter eq(String columnName, Object value) {
+ return new ValueEq(getIdColumnIndex(columnName), (String) value);
+ }
+
+ @TsFileApi
+ public Filter neq(String columnName, Object value) {
+ return new ValueNotEq(getIdColumnIndex(columnName), (String) value);
+ }
+
+ @TsFileApi
+ public Filter lt(String columnName, Object value) {
+ return new TagFilterOperators.ValueLt(getIdColumnIndex(columnName), (String) value);
+ }
+
+ @TsFileApi
+ public Filter lteq(String columnName, Object value) {
+ return new TagFilterOperators.ValueLtEq(getIdColumnIndex(columnName), (String) value);
+ }
+
+ @TsFileApi
+ public Filter gt(String columnName, Object value) {
+ return new TagFilterOperators.ValueGt(getIdColumnIndex(columnName), (String) value);
+ }
+
+ @TsFileApi
+ public Filter gteq(String columnName, Object value) {
+ return new TagFilterOperators.ValueGtEq(getIdColumnIndex(columnName), (String) value);
+ }
+
+ @TsFileApi
+ public Filter betweenAnd(String columnName, Object value1, Object value2) {
+ return new TagFilterOperators.ValueBetweenAnd(
+ getIdColumnIndex(columnName), (String) value1, (String) value2);
+ }
+
+ @TsFileApi
+ public Filter notBetweenAnd(String columnName, Object value1, Object value2) {
+ return new TagFilterOperators.ValueNotBetweenAnd(
+ getIdColumnIndex(columnName), (String) value1, (String) value2);
+ }
+
+ @TsFileApi
+ public Filter regExp(String columnName, String pattern) {
+ return new TagFilterOperators.ValueRegexp(
+ getIdColumnIndex(columnName), Pattern.compile(pattern));
+ }
+
+ @TsFileApi
+ public Filter notRegExp(String columnName, String pattern) {
+ return new TagFilterOperators.ValueNotRegexp(
+ getIdColumnIndex(columnName), Pattern.compile(pattern));
+ }
+
+ @TsFileApi
+ public Filter like(String columnName, String pattern) {
+ return new TagFilterOperators.ValueLike(
+ getIdColumnIndex(columnName), LikePattern.compile(pattern, Optional.empty()));
+ }
+
+ @TsFileApi
+ public Filter notLike(String columnName, String pattern) {
+ return new TagFilterOperators.ValueNotLike(
+ getIdColumnIndex(columnName), LikePattern.compile(pattern, Optional.empty()));
+ }
+
+ @TsFileApi
+ public Filter and(Filter left, Filter right) {
+ return new And(left, right);
+ }
+
+ @TsFileApi
+ public Filter or(Filter left, Filter right) {
+ return new Or(left, right);
+ }
+
+ @TsFileApi
+ public Filter not(Filter value) {
+ return new Not(value);
+ }
+}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/operator/And.java b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/operator/And.java
index de0f3381..8df4e509 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/operator/And.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/operator/And.java
@@ -77,6 +77,11 @@ public class And extends BinaryLogicalFilter {
return left.satisfyBinary(time, value) && right.satisfyBinary(time, value);
}
+ @Override
+ public boolean satisfyString(long time, String value) {
+ return left.satisfyString(time, value) && right.satisfyString(time, value);
+ }
+
@Override
public boolean satisfyRow(long time, Object[] values) {
return left.satisfyRow(time, values) && right.satisfyRow(time, values);
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/operator/Not.java b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/operator/Not.java
index 00918915..e02a3697 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/operator/Not.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/operator/Not.java
@@ -90,6 +90,11 @@ public class Not extends Filter {
return !filter.satisfyRow(time, values);
}
+ @Override
+ public boolean satisfyString(long time, String value) {
+ return !filter.satisfyString(time, value);
+ }
+
@Override
public boolean satisfyBooleanRow(long time, boolean[] values) {
return !filter.satisfyBooleanRow(time, values);
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/operator/Or.java b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/operator/Or.java
index f0f1c2f1..97a78758 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/operator/Or.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/operator/Or.java
@@ -77,6 +77,11 @@ public class Or extends BinaryLogicalFilter {
return left.satisfyBinary(time, value) || right.satisfyBinary(time, value);
}
+ @Override
+ public boolean satisfyString(long time, String value) {
+ return left.satisfyString(time, value) || right.satisfyString(time, value);
+ }
+
@Override
public boolean satisfyRow(long time, Object[] values) {
return left.satisfyRow(time, values) || right.satisfyRow(time, values);
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/ResultSet.java b/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/ResultSet.java
index 0f2d72bf..2e5d0706 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/ResultSet.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/ResultSet.java
@@ -20,9 +20,11 @@
package org.apache.tsfile.read.query.dataset;
import org.apache.tsfile.annotations.TsFileApi;
+import org.apache.tsfile.write.record.TSRecord;
import java.io.IOException;
import java.time.LocalDate;
+import java.util.Iterator;
public interface ResultSet extends AutoCloseable {
@@ -87,5 +89,8 @@ public interface ResultSet extends AutoCloseable {
boolean isNull(int columnIndex);
@TsFileApi
- public abstract void close();
+ void close();
+
+ @TsFileApi
+ Iterator<TSRecord> recordIterator();
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/TableResultSet.java b/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/TableResultSet.java
index d13439b4..bd908d7e 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/TableResultSet.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/TableResultSet.java
@@ -23,25 +23,38 @@ import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.read.common.block.TsBlock;
import org.apache.tsfile.read.reader.block.TsBlockReader;
import org.apache.tsfile.utils.DateUtils;
+import org.apache.tsfile.write.record.TSRecord;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.time.LocalDate;
+import java.util.Iterator;
import java.util.List;
+import java.util.NoSuchElementException;
public class TableResultSet extends AbstractResultSet {
+
private static final Logger LOG = LoggerFactory.getLogger(TableResultSet.class);
private final TsBlockReader tsBlockReader;
private TsBlock currentTsBlock;
private int currentTsBlockIndex;
+ private final List<String> columnNameList;
+ private final List<TSDataType> dataTypes;
+ private final String tableName;
public TableResultSet(
- TsBlockReader tsBlockReader, List<String> columnNameList, List<TSDataType> dataTypeList) {
+ TsBlockReader tsBlockReader,
+ List<String> columnNameList,
+ List<TSDataType> dataTypeList,
+ String tableName) {
super(columnNameList, dataTypeList);
+ this.columnNameList = columnNameList;
+ this.dataTypes = dataTypeList;
this.tsBlockReader = tsBlockReader;
+ this.tableName = tableName;
}
@Override
@@ -129,4 +142,89 @@ public class TableResultSet extends AbstractResultSet {
LOG.error("Failed to close tsBlockReader");
}
}
+
+ @Override
+ public Iterator<TSRecord> recordIterator() {
+ return new RecordIterator();
+ }
+
+ private class RecordIterator implements Iterator<TSRecord> {
+
+ private TSRecord cachedRecord = null;
+ private boolean exhausted = false;
+
+ @Override
+ public boolean hasNext() {
+ if (cachedRecord != null) {
+ return true;
+ }
+ if (exhausted) {
+ return false;
+ }
+
+ try {
+ return cacheNextRecord();
+ } catch (IOException e) {
+ throw new NoSuchElementException(e.toString());
+ }
+ }
+
+ private boolean cacheNextRecord() throws IOException {
+ boolean next = TableResultSet.this.next();
+ if (!next) {
+ exhausted = true;
+ return false;
+ }
+ cachedRecord = new TSRecord(tableName, getLong("Time"));
+ for (int i = 0; i < dataTypes.size(); i++) {
+ TSDataType dataType = dataTypes.get(i);
+ String columnName = columnNameList.get(i);
+ if (isNull(columnName)) {
+ cachedRecord.dataPointList.add(null);
+ continue;
+ }
+ switch (dataType) {
+ case INT32:
+ case DATE:
+ cachedRecord.addPoint(columnName, getInt(columnName));
+ break;
+ case INT64:
+ case TIMESTAMP:
+ cachedRecord.addPoint(columnName, getLong(columnName));
+ break;
+ case FLOAT:
+ cachedRecord.addPoint(columnName, getFloat(columnName));
+ break;
+ case DOUBLE:
+ cachedRecord.addPoint(columnName, getDouble(columnName));
+ break;
+ case STRING:
+ case TEXT:
+ cachedRecord.addPoint(columnName, getString(columnName));
+ break;
+ case BLOB:
+ cachedRecord.addPoint(columnName, getBinary(columnName));
+ break;
+ case BOOLEAN:
+ cachedRecord.addPoint(columnName, getBoolean(columnName));
+ break;
+ case VECTOR:
+ case UNKNOWN:
+ default:
+ break;
+ }
+ }
+ return true;
+ }
+
+ @Override
+ public TSRecord next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ TSRecord ret = cachedRecord;
+ cachedRecord = null;
+ return ret;
+ }
+ }
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/TreeResultSet.java b/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/TreeResultSet.java
index 0f3aade6..2b260167 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/TreeResultSet.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/TreeResultSet.java
@@ -20,19 +20,48 @@
package org.apache.tsfile.read.query.dataset;
import org.apache.tsfile.annotations.TsFileApi;
+import org.apache.tsfile.file.metadata.StringArrayDeviceID;
+import org.apache.tsfile.read.common.Field;
import org.apache.tsfile.read.common.Path;
+import org.apache.tsfile.write.record.TSRecord;
import java.io.IOException;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
import java.util.stream.Collectors;
+import java.util.stream.IntStream;
public class TreeResultSet extends AbstractResultSet {
private QueryDataSet queryDataSet;
+ private List<String> deviceList;
+ private List<String> measurementList;
+ private Map<Path, Integer> pathIndexMap;
+ private Map<String, Map<String, Path>> cachedPaths;
- public TreeResultSet(QueryDataSet queryDataSet) {
+ public TreeResultSet(
+ QueryDataSet queryDataSet, List<String> deviceIds, List<String> measurementNames) {
super(
queryDataSet.getPaths().stream().map(Path::toString).collect(Collectors.toList()),
queryDataSet.getDataTypes());
this.queryDataSet = queryDataSet;
+ this.deviceList = deviceIds;
+ this.measurementList = measurementNames;
+ List<Path> paths = queryDataSet.getPaths();
+ this.pathIndexMap =
+ IntStream.range(0, paths.size()).boxed().collect(Collectors.toMap(paths::get, i -> i));
+ this.cachedPaths = new HashMap<>();
+ for (String device : deviceList) {
+ Map<String, Path> measurementPathMap = new HashMap<>();
+ for (String measurement : measurementList) {
+ measurementPathMap.put(
+ measurement, new Path(new StringArrayDeviceID(device), measurement, false));
+ }
+ cachedPaths.put(device, measurementPathMap);
+ }
}
@TsFileApi
@@ -51,4 +80,96 @@ public class TreeResultSet extends AbstractResultSet {
public void close() {
// nothing to be done
}
+
+ @Override
+ public Iterator<TSRecord> recordIterator() {
+ return new TreeResultSet.RecordIterator();
+ }
+
+ private class RecordIterator implements Iterator<TSRecord> {
+ private final LinkedList<TSRecord> recordBuffer = new LinkedList<>();
+ private boolean exhausted = false;
+
+ @Override
+ public boolean hasNext() {
+ if (!recordBuffer.isEmpty()) {
+ return true;
+ }
+ if (exhausted) {
+ return false;
+ }
+
+ try {
+ return fetchRecords();
+ } catch (IOException e) {
+ throw new NoSuchElementException(e.toString());
+ }
+ }
+
+ private boolean fetchRecords() throws IOException {
+ boolean hasNewRecords = false;
+ while (TreeResultSet.this.next()) {
+ for (String device : deviceList) {
+ TSRecord record = new TSRecord(device, getLong("Time"));
+ record.addPoint("id", device);
+
+ Map<String, Path> devicePaths = cachedPaths.get(device);
+ for (String measurement : measurementList) {
+ Path path = devicePaths.get(measurement);
+ Integer pathIdx = pathIndexMap.get(path);
+ if (pathIdx != null) {
+ Field field = currentRow.getField(pathIdx);
+ switch (field.getDataType()) {
+ case INT32:
+ case DATE:
+ record.addPoint(measurement, field.getIntV());
+ break;
+ case INT64:
+ case TIMESTAMP:
+ record.addPoint(measurement, field.getLongV());
+ break;
+ case FLOAT:
+ record.addPoint(measurement, field.getFloatV());
+ break;
+ case DOUBLE:
+ record.addPoint(measurement, field.getDoubleV());
+ break;
+ case STRING:
+ case TEXT:
+ record.addPoint(measurement, field.getStringValue());
+ break;
+ case BLOB:
+ record.addPoint(measurement, field.getBinaryV().getValues());
+ break;
+ case BOOLEAN:
+ record.addPoint(measurement, field.getBoolV());
+ break;
+ case VECTOR:
+ case UNKNOWN:
+ default:
+ break;
+ }
+ } else {
+ record.dataPointList.add(null);
+ }
+ }
+ recordBuffer.add(record);
+ hasNewRecords = true;
+ }
+ }
+ if (!hasNewRecords) {
+ exhausted = true;
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public TSRecord next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ return recordBuffer.poll();
+ }
+ }
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/query/executor/TableQueryExecutor.java b/java/tsfile/src/main/java/org/apache/tsfile/read/query/executor/TableQueryExecutor.java
index a192a360..c50d1bf1 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/query/executor/TableQueryExecutor.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/query/executor/TableQueryExecutor.java
@@ -29,6 +29,7 @@ import org.apache.tsfile.file.metadata.TsFileMetadata;
import org.apache.tsfile.read.controller.IChunkLoader;
import org.apache.tsfile.read.controller.IMetadataQuerier;
import org.apache.tsfile.read.expression.ExpressionTree;
+import org.apache.tsfile.read.filter.basic.Filter;
import org.apache.tsfile.read.query.executor.task.DeviceTaskIterator;
import org.apache.tsfile.read.reader.block.DeviceOrderedTsBlockReader;
import org.apache.tsfile.read.reader.block.TsBlockReader;
@@ -62,7 +63,7 @@ public class TableQueryExecutor {
* @param tableName table to query
* @param columns columns to query (ID or MEASUREMENT)
* @param timeFilter time predicate
- * @param idFilter id predicate
+ * @param tagFilter id predicate
* @param measurementFilter measurement predicate
* @return an iterator of TsBlocks
* @throws ReadProcessException if the read process fails
@@ -71,7 +72,7 @@ public class TableQueryExecutor {
String tableName,
List<String> columns,
ExpressionTree timeFilter,
- ExpressionTree idFilter,
+ Filter tagFilter,
ExpressionTree measurementFilter)
throws ReadProcessException {
TsFileMetadata fileMetadata = metadataQuerier.getWholeFileMetadata();
@@ -90,7 +91,7 @@ public class TableQueryExecutor {
DeviceTaskIterator deviceTaskIterator =
new DeviceTaskIterator(
- columns, tableRoot, columnMapping, metadataQuerier, idFilter, tableSchema);
+ columns, tableRoot, columnMapping, metadataQuerier, tagFilter, tableSchema);
switch (tableQueryOrdering) {
case DEVICE:
return new DeviceOrderedTsBlockReader(
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/query/executor/task/DeviceTaskIterator.java b/java/tsfile/src/main/java/org/apache/tsfile/read/query/executor/task/DeviceTaskIterator.java
index 8ee44650..399cd32c 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/query/executor/task/DeviceTaskIterator.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/query/executor/task/DeviceTaskIterator.java
@@ -23,7 +23,7 @@ import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.file.metadata.MetadataIndexNode;
import org.apache.tsfile.file.metadata.TableSchema;
import org.apache.tsfile.read.controller.IMetadataQuerier;
-import org.apache.tsfile.read.expression.ExpressionTree;
+import org.apache.tsfile.read.filter.basic.Filter;
import org.apache.tsfile.read.query.executor.TableQueryExecutor.ColumnMapping;
import org.apache.tsfile.utils.Pair;
@@ -41,11 +41,11 @@ public class DeviceTaskIterator implements Iterator<DeviceQueryTask> {
MetadataIndexNode indexRoot,
ColumnMapping columnMapping,
IMetadataQuerier metadataQuerier,
- ExpressionTree idFilter,
+ Filter tagFilter,
TableSchema tableSchema) {
this.columnNames = columnNames;
this.columnMapping = columnMapping;
- this.deviceMetaIterator = metadataQuerier.deviceIterator(indexRoot, idFilter);
+ this.deviceMetaIterator = metadataQuerier.deviceIterator(indexRoot, tagFilter);
this.tableSchema = tableSchema;
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/v4/DeviceTableModelReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/v4/DeviceTableModelReader.java
index 894353fa..927ac895 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/v4/DeviceTableModelReader.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/v4/DeviceTableModelReader.java
@@ -31,6 +31,7 @@ import org.apache.tsfile.read.controller.IChunkLoader;
import org.apache.tsfile.read.controller.IMetadataQuerier;
import org.apache.tsfile.read.controller.MetadataQuerierByFileImpl;
import org.apache.tsfile.read.expression.ExpressionTree;
+import org.apache.tsfile.read.filter.basic.Filter;
import org.apache.tsfile.read.query.dataset.ResultSet;
import org.apache.tsfile.read.query.dataset.TableResultSet;
import org.apache.tsfile.read.query.executor.TableQueryExecutor;
@@ -79,6 +80,13 @@ public class DeviceTableModelReader implements ITsFileReader {
@TsFileApi
public ResultSet query(String tableName, List<String> columnNames, long startTime, long endTime)
throws IOException, NoTableException, NoMeasurementException, ReadProcessException {
+ return query(tableName, columnNames, startTime, endTime, null);
+ }
+
+ @Override
+ public ResultSet query(
+ String tableName, List<String> columnNames, long startTime, long endTime, Filter tagFilter)
+ throws ReadProcessException, IOException, NoTableException, NoMeasurementException {
String lowerCaseTableName = tableName.toLowerCase();
TableSchema tableSchema = fileReader.getTableSchemaMap().get(lowerCaseTableName);
if (tableSchema == null) {
@@ -100,9 +108,9 @@ public class DeviceTableModelReader implements ITsFileReader {
lowerCaseTableName,
lowerCaseColumnNames,
new ExpressionTree.TimeBetweenAnd(startTime, endTime),
- null,
+ tagFilter,
null);
- return new TableResultSet(tsBlockReader, columnNames, dataTypeList);
+ return new TableResultSet(tsBlockReader, columnNames, dataTypeList, tableName);
}
@Override
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/v4/ITsFileReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/v4/ITsFileReader.java
index 885e8452..995f73f6 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/v4/ITsFileReader.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/v4/ITsFileReader.java
@@ -24,6 +24,7 @@ import org.apache.tsfile.exception.read.ReadProcessException;
import org.apache.tsfile.exception.write.NoMeasurementException;
import org.apache.tsfile.exception.write.NoTableException;
import org.apache.tsfile.file.metadata.TableSchema;
+import org.apache.tsfile.read.filter.basic.Filter;
import org.apache.tsfile.read.query.dataset.ResultSet;
import java.io.IOException;
@@ -36,6 +37,11 @@ public interface ITsFileReader extends AutoCloseable {
ResultSet query(String tableName, List<String> columnNames, long startTime, long endTime)
throws ReadProcessException, IOException, NoTableException, NoMeasurementException;
+ @TsFileApi
+ ResultSet query(
+ String tableName, List<String> columnNames, long startTime, long endTime, Filter tagFilter)
+ throws ReadProcessException, IOException, NoTableException, NoMeasurementException;
+
@TsFileApi
Optional<TableSchema> getTableSchemas(String tableName) throws IOException;
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/v4/ITsFileReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/v4/ITsFileTreeReader.java
similarity index 57%
copy from java/tsfile/src/main/java/org/apache/tsfile/read/v4/ITsFileReader.java
copy to java/tsfile/src/main/java/org/apache/tsfile/read/v4/ITsFileTreeReader.java
index 885e8452..202553ea 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/v4/ITsFileReader.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/v4/ITsFileTreeReader.java
@@ -19,29 +19,37 @@
package org.apache.tsfile.read.v4;
+import org.apache.tsfile.annotations.TreeModel;
import org.apache.tsfile.annotations.TsFileApi;
-import org.apache.tsfile.exception.read.ReadProcessException;
-import org.apache.tsfile.exception.write.NoMeasurementException;
-import org.apache.tsfile.exception.write.NoTableException;
-import org.apache.tsfile.file.metadata.TableSchema;
import org.apache.tsfile.read.query.dataset.ResultSet;
+import org.apache.tsfile.write.schema.MeasurementSchema;
import java.io.IOException;
import java.util.List;
-import java.util.Optional;
-public interface ITsFileReader extends AutoCloseable {
+/** New tree-model read interface for TsFile. */
+public interface ITsFileTreeReader extends AutoCloseable {
+ /** Execute a query and return a ResultSet wrapper. */
@TsFileApi
- ResultSet query(String tableName, List<String> columnNames, long startTime, long endTime)
- throws ReadProcessException, IOException, NoTableException, NoMeasurementException;
+ @TreeModel
+ ResultSet query(
+ List<String> deviceIds, List<String> measurementNames, long startTime, long endTime)
+ throws IOException;
+ /** Return all device IDs found in the file. */
@TsFileApi
- Optional<TableSchema> getTableSchemas(String tableName) throws IOException;
+ @TreeModel
+ List<String> getAllDeviceIds() throws IOException;
+ /** Return measurement schema list for a given device. */
@TsFileApi
- List<TableSchema> getAllTableSchema() throws IOException;
+ @TreeModel
+ List<MeasurementSchema> getDeviceSchema(String deviceId) throws IOException;
+ /** Close underlying resources. */
@TsFileApi
- void close();
+ @TreeModel
+ @Override
+ void close() throws IOException;
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/v4/TsFileTreeReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/v4/TsFileTreeReader.java
new file mode 100644
index 00000000..fe7a0c35
--- /dev/null
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/v4/TsFileTreeReader.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tsfile.read.v4;
+
+import org.apache.tsfile.annotations.TreeModel;
+import org.apache.tsfile.annotations.TsFileApi;
+import org.apache.tsfile.file.metadata.StringArrayDeviceID;
+import org.apache.tsfile.read.TsFileReader;
+import org.apache.tsfile.read.common.Path;
+import org.apache.tsfile.read.expression.IExpression;
+import org.apache.tsfile.read.expression.QueryExpression;
+import org.apache.tsfile.read.expression.impl.GlobalTimeExpression;
+import org.apache.tsfile.read.filter.operator.TimeFilterOperators;
+import org.apache.tsfile.read.query.dataset.QueryDataSet;
+import org.apache.tsfile.read.query.dataset.ResultSet;
+import org.apache.tsfile.read.query.dataset.TreeResultSet;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * TsFileTreeReader is an implementation of ITsFileTreeReader that provides query capability for
+ * TsFile with a tree-style interface. It internally wraps {@link TsFileReader}.
+ */
+public class TsFileTreeReader implements ITsFileTreeReader {
+ private final TsFileReader tsfileReader;
+
+ /**
+ * Construct a TsFileTreeReader with the given TsFile.
+ *
+ * @param file the TsFile to read
+ * @throws IOException if an I/O error occurs during file opening
+ */
+ @TsFileApi
+ @TreeModel
+ public TsFileTreeReader(File file) throws IOException {
+ tsfileReader = new TsFileReader(file);
+ }
+
+ /**
+ * Execute a query on the given devices and measurements within the specified time range.
+ *
+ * @param deviceIds list of device IDs to query
+ * @param measurementNames list of measurement names to query
+ * @param startTime query start timestamp (inclusive)
+ * @param endTime query end timestamp (inclusive)
+ * @return a {@link ResultSet} containing the query results
+ * @throws IOException if an I/O error occurs during query execution
+ */
+ @TsFileApi
+ @TreeModel
+ @Override
+ public ResultSet query(
+ List<String> deviceIds, List<String> measurementNames, long startTime, long endTime)
+ throws IOException {
+ List<Path> paths = new ArrayList<>();
+ for (String deviceId : deviceIds) {
+ for (String measurementName : measurementNames) {
+ paths.add(new Path(deviceId, measurementName, true));
+ }
+ }
+ IExpression expression =
+ new GlobalTimeExpression(new TimeFilterOperators.TimeBetweenAnd(startTime, endTime));
+ QueryExpression queryExpression = QueryExpression.create(paths, expression);
+ QueryDataSet queryDataSet = tsfileReader.query(queryExpression);
+ return new TreeResultSet(queryDataSet, deviceIds, measurementNames);
+ }
+
+ /**
+ * Get all device IDs existing in the TsFile.
+ *
+ * @return list of all device IDs
+ * @throws IOException if an I/O error occurs during metadata fetching
+ */
+ @TsFileApi
+ @TreeModel
+ @Override
+ public List<String> getAllDeviceIds() throws IOException {
+ return tsfileReader.getAllDeviceIds();
+ }
+
+ /**
+ * Get the measurement schema of the given device.
+ *
+ * @param deviceId the target device ID
+ * @return list of {@link MeasurementSchema} for the device
+ * @throws IOException if an I/O error occurs during schema fetching
+ */
+ @TsFileApi
+ @TreeModel
+ @Override
+ public List<MeasurementSchema> getDeviceSchema(String deviceId) throws IOException {
+ return tsfileReader.getMeasurement(new StringArrayDeviceID(deviceId));
+ }
+
+ /**
+ * Close the TsFileTreeReader and release resources.
+ *
+ * @throws IOException if an I/O error occurs during closing
+ */
+ @TsFileApi
+ @TreeModel
+ @Override
+ public void close() throws IOException {
+ tsfileReader.close();
+ }
+}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/ITsFileWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/read/v4/TsFileTreeReaderBuilder.java
similarity index 58%
copy from java/tsfile/src/main/java/org/apache/tsfile/write/v4/ITsFileWriter.java
copy to java/tsfile/src/main/java/org/apache/tsfile/read/v4/TsFileTreeReaderBuilder.java
index 8610ee1f..15cd7f8d 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/ITsFileWriter.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/v4/TsFileTreeReaderBuilder.java
@@ -17,19 +17,36 @@
* under the License.
*/
-package org.apache.tsfile.write.v4;
+package org.apache.tsfile.read.v4;
+import org.apache.tsfile.annotations.TreeModel;
import org.apache.tsfile.annotations.TsFileApi;
-import org.apache.tsfile.exception.write.WriteProcessException;
-import org.apache.tsfile.write.record.Tablet;
+import java.io.File;
import java.io.IOException;
-public interface ITsFileWriter extends AutoCloseable {
+/** Builder for TsFileTreeReader. */
+public class TsFileTreeReaderBuilder {
+ private File file;
@TsFileApi
- void write(Tablet tablet) throws IOException, WriteProcessException;
+ @TreeModel
+ public TsFileTreeReaderBuilder file(File file) {
+ this.file = file;
+ return this;
+ }
+ /**
+ * Build an ITsFileTreeReader instance.
+ *
+ * @throws IllegalStateException if required fields are missing.
+ */
@TsFileApi
- void close();
+ @TreeModel
+ public ITsFileTreeReader build() throws IOException {
+ if (this.file == null) {
+ throw new IllegalStateException("file must be set before build()");
+ }
+ return new TsFileTreeReader(this.file);
+ }
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteIOUtils.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteIOUtils.java
index 6e38e513..e6b72e40 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteIOUtils.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteIOUtils.java
@@ -997,6 +997,18 @@ public class ReadWriteIOUtils {
return set;
}
+ public static Set<String> readStringSet(ByteBuffer buffer) {
+ int size = readInt(buffer);
+ if (size <= 0) {
+ return Collections.emptySet();
+ }
+ Set<String> set = new HashSet<>();
+ for (int i = 0; i < size; i++) {
+ set.add(readString(buffer));
+ }
+ return set;
+ }
+
// read object set with self define length
public static <T> Set<T> readObjectSet(ByteBuffer buffer) {
int size = readInt(buffer);
@@ -1082,6 +1094,16 @@ public class ReadWriteIOUtils {
}
}
+ public static void writeStringSet(Set<String> set, DataOutputStream outputStream)
+ throws IOException {
+ write(set.contains(null) ? set.size() - 1 : set.size(), outputStream);
+ for (String e : set) {
+ if (e != null) {
+ write(e, outputStream);
+ }
+ }
+ }
+
public static CompressionType readCompressionType(InputStream inputStream) throws IOException {
byte n = readByte(inputStream);
return CompressionType.deserialize(n);
@@ -1145,6 +1167,7 @@ public class ReadWriteIOUtils {
BINARY,
BOOLEAN,
STRING,
+ TAG,
NULL
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/TsFileWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/TsFileWriter.java
index 08ca25ce..7223e432 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/write/TsFileWriter.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/write/TsFileWriter.java
@@ -256,6 +256,25 @@ public class TsFileWriter implements AutoCloseable {
this.chunkGroupSizeThreshold = chunkGroupSizeThreshold;
}
+ /**
+ * Sets the memory threshold for flushing data chunks.
+ *
+ * @param memoryThreshold the memory threshold in bytes
+ * @throws IOException if the page size is greater than or equal to the new threshold
+ */
+ public void setMemoryThreshold(int memoryThreshold) throws IOException {
+ this.chunkGroupSizeThreshold = memoryThreshold;
+ if (this.pageSize >= chunkGroupSizeThreshold) {
+ String errorMsg =
+ String.format(
+ "Invalid memory threshold configuration: page size %d must be smaller than chunk group size %d. "
+ + "Please either increase the chunk group size or decrease the page size.",
+ pageSize, chunkGroupSizeThreshold);
+ LOG.error(errorMsg);
+ throw new IOException(errorMsg);
+ }
+ }
+
public void registerSchemaTemplate(
String templateName, Map<String, IMeasurementSchema> template, boolean isAligned) {
getSchema().registerSchemaTemplate(templateName, new MeasurementGroup(isAligned, template));
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java
index 3691f737..3207c409 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java
@@ -167,6 +167,9 @@ public class AlignedChunkGroupWriterImpl implements IChunkGroupWriter {
? point.getMeasurementId().toLowerCase()
: point.getMeasurementId();
ValueChunkWriter valueChunkWriter = valueChunkWriterMap.get(measurementId);
+ if (valueChunkWriter == null) {
+ valueChunkWriter = tryToAddSeriesWriterInternal(point.getMeasurementSchema());
+ }
switch (point.getType()) {
case BOOLEAN:
valueChunkWriter.write(time, (boolean) point.getValue(), isNull);
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/record/TSRecord.java b/java/tsfile/src/main/java/org/apache/tsfile/write/record/TSRecord.java
index b7c24831..9f36651d 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/write/record/TSRecord.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/write/record/TSRecord.java
@@ -18,10 +18,15 @@
*/
package org.apache.tsfile.write.record;
+import org.apache.tsfile.annotations.TableModel;
import org.apache.tsfile.annotations.TsFileApi;
import org.apache.tsfile.common.conf.TSFileConfig;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.exception.write.ConflictDataTypeException;
+import org.apache.tsfile.exception.write.WriteProcessException;
import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.file.metadata.IDeviceID.Factory;
+import org.apache.tsfile.file.metadata.TableSchema;
import org.apache.tsfile.utils.Binary;
import org.apache.tsfile.utils.StringContainer;
import org.apache.tsfile.write.record.datapoint.BooleanDataPoint;
@@ -49,19 +54,22 @@ public class TSRecord {
/** deviceId of this TSRecord. */
public IDeviceID deviceId;
+ private String tableName;
+
/** all value of this TSRecord. */
public List<DataPoint> dataPointList = new ArrayList<>();
/**
* constructor of TSRecord.
*
- * @param deviceId deviceId of this TSRecord
+ * @param deviceIdOrTableName deviceId of this TSRecord
* @param timestamp timestamp of this TSRecord
*/
@TsFileApi
- public TSRecord(String deviceId, long timestamp) {
+ public TSRecord(String deviceIdOrTableName, long timestamp) {
this.time = timestamp;
- this.deviceId = Factory.DEFAULT_FACTORY.create(deviceId);
+ this.deviceId = Factory.DEFAULT_FACTORY.create(deviceIdOrTableName);
+ this.tableName = deviceIdOrTableName;
}
@TsFileApi
@@ -137,9 +145,38 @@ public class TSRecord {
StringContainer sc = new StringContainer(" ");
sc.addTail("{device id:", deviceId, "time:", time, ",data:[");
for (DataPoint tuple : dataPointList) {
- sc.addTail(tuple);
+ if (tuple != null) {
+ sc.addTail(tuple);
+ } else {
+ sc.addTail("{ null }");
+ }
}
sc.addTail("]}");
return sc.toString();
}
+
+ @TableModel
+ public IDeviceID getDeviceId(TableSchema schema) throws WriteProcessException {
+ int tagCnt = schema.getTagColumnCnt();
+ String[] idSegments = new String[tagCnt + 1];
+ idSegments[0] = schema.getTableName();
+
+ for (DataPoint dataPoint : dataPointList) {
+ String columnName = dataPoint.getMeasurementId();
+ int idColumnOrder = schema.findIdColumnOrder(columnName);
+ if (idColumnOrder != -1) {
+ if (!(dataPoint instanceof StringDataPoint)) {
+ throw new ConflictDataTypeException(dataPoint.getType(), TSDataType.STRING);
+ }
+ Object value = dataPoint.getValue();
+ idSegments[idColumnOrder + 1] = value != null ? value.toString() : null;
+ }
+ }
+
+ return Factory.DEFAULT_FACTORY.create(idSegments);
+ }
+
+ public String getTableName() {
+ return tableName.toLowerCase();
+ }
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/record/datapoint/DataPoint.java b/java/tsfile/src/main/java/org/apache/tsfile/write/record/datapoint/DataPoint.java
index 80e617ca..7bd126c5 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/write/record/datapoint/DataPoint.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/write/record/datapoint/DataPoint.java
@@ -26,6 +26,7 @@ import org.apache.tsfile.utils.DateUtils;
import org.apache.tsfile.utils.StringContainer;
import org.apache.tsfile.write.UnSupportedDataTypeException;
import org.apache.tsfile.write.chunk.ChunkWriterImpl;
+import org.apache.tsfile.write.schema.IMeasurementSchema;
import java.io.IOException;
import java.time.LocalDate;
@@ -43,6 +44,8 @@ public abstract class DataPoint {
/** measurementId of this DataPoint. */
protected final String measurementId;
+ protected IMeasurementSchema measurementSchema;
+
/**
* constructor of DataPoint.
*
@@ -157,4 +160,12 @@ public abstract class DataPoint {
public void setDate(LocalDate value) {
throw new UnsupportedOperationException("set Date not support in DataPoint");
}
+
+ public IMeasurementSchema getMeasurementSchema() {
+ return measurementSchema;
+ }
+
+ public void setMeasurementSchema(IMeasurementSchema measurementSchema) {
+ this.measurementSchema = measurementSchema;
+ }
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/schema/MeasurementSchemaBuilder.java b/java/tsfile/src/main/java/org/apache/tsfile/write/schema/MeasurementSchemaBuilder.java
new file mode 100644
index 00000000..b523a339
--- /dev/null
+++ b/java/tsfile/src/main/java/org/apache/tsfile/write/schema/MeasurementSchemaBuilder.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tsfile.write.schema;
+
+import org.apache.tsfile.common.conf.TSFileDescriptor;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.enums.CompressionType;
+import org.apache.tsfile.file.metadata.enums.TSEncoding;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * A builder class for constructing {@link MeasurementSchema} instances.
+ *
+ * <p>This builder provides a fluent API for setting various properties of a measurement schema,
+ * including required fields (name and data type) and optional fields (encoding and compression).
+ *
+ * <p>Example usage:
+ *
+ * <pre>{@code
+ * MeasurementSchema schema = new MeasurementSchemaBuilder("temperature", TSDataType.FLOAT)
+ * .withEncoding(TSEncoding.RLE)
+ * .withCompression(CompressionType.SNAPPY)
+ * .build();
+ * }</pre>
+ */
+public class MeasurementSchemaBuilder {
+ private String measurementName;
+ private TSDataType dataType;
+ private TSEncoding encoding;
+ private CompressionType compressionType;
+ private Map<String, String> props;
+
+ /**
+ * Creates a new builder with the required fields.
+ *
+ * @param measurementName the name of the measurement (cannot be null or empty)
+ * @param dataType the data type of the measurement (cannot be null)
+ * @throws IllegalArgumentException if required parameters are null or empty
+ */
+ public MeasurementSchemaBuilder(String measurementName, TSDataType dataType) {
+ if (measurementName == null || measurementName.trim().isEmpty()) {
+ throw new IllegalArgumentException("Measurement name cannot be null or empty");
+ }
+ if (dataType == null) {
+ throw new IllegalArgumentException("Data type cannot be null");
+ }
+
+ this.measurementName = measurementName;
+ this.dataType = dataType;
+ // Set default values from TSFile configuration
+ this.encoding =
+ TSEncoding.valueOf(TSFileDescriptor.getInstance().getConfig().getValueEncoder(dataType));
+ this.compressionType = TSFileDescriptor.getInstance().getConfig().getCompressor(dataType);
+ }
+
+ /**
+ * Sets the encoding type for the measurement.
+ *
+ * @param encoding the encoding type (optional, defaults based on data type)
+ * @return this builder instance for method chaining
+ */
+ public MeasurementSchemaBuilder withEncoding(TSEncoding encoding) {
+ this.encoding = encoding;
+ return this;
+ }
+
+ /**
+ * Sets the compression type for the measurement.
+ *
+ * @param compressionType the compression type (optional, defaults to LZ4)
+ * @return this builder instance for method chaining
+ */
+ public MeasurementSchemaBuilder withCompression(CompressionType compressionType) {
+ this.compressionType = compressionType;
+ return this;
+ }
+
+ /**
+ * Adds a property to the measurement schema.
+ *
+ * @param key the property key (cannot be null)
+ * @param value the property value (cannot be null)
+ * @return this builder instance for method chaining
+ * @throws IllegalArgumentException if key or value is null
+ */
+ public MeasurementSchemaBuilder withProperty(String key, String value) {
+ if (key == null || value == null) {
+ throw new IllegalArgumentException("Property key and value cannot be null");
+ }
+ if (this.props == null) {
+ this.props = new HashMap<>();
+ }
+ this.props.put(key, value);
+ return this;
+ }
+
+ /**
+ * Sets multiple properties for the measurement schema.
+ *
+ * @param props a map of properties (can be null)
+ * @return this builder instance for method chaining
+ */
+ public MeasurementSchemaBuilder withProperties(Map<String, String> props) {
+ this.props = props;
+ return this;
+ }
+
+ /**
+ * Builds the final {@link MeasurementSchema} instance.
+ *
+ * @return a new MeasurementSchema configured with the builder's settings
+ */
+ public MeasurementSchema build() {
+ return new MeasurementSchema(measurementName, dataType, encoding, compressionType, props);
+ }
+}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/DeviceTableModelWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/DeviceTableModelWriter.java
index f64f285f..b2ae6b1f 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/DeviceTableModelWriter.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/DeviceTableModelWriter.java
@@ -30,7 +30,9 @@ import org.apache.tsfile.file.metadata.TableSchema;
import org.apache.tsfile.utils.Pair;
import org.apache.tsfile.utils.WriteUtils;
import org.apache.tsfile.write.chunk.AlignedChunkGroupWriterImpl;
+import org.apache.tsfile.write.record.TSRecord;
import org.apache.tsfile.write.record.Tablet;
+import org.apache.tsfile.write.record.datapoint.DataPoint;
import org.apache.tsfile.write.schema.IMeasurementSchema;
import java.io.File;
@@ -76,6 +78,41 @@ public class DeviceTableModelWriter extends AbstractTableModelTsFileWriter {
checkMemorySizeAndMayFlushChunks();
}
+ @TsFileApi
+ @Override
+ public void write(TSRecord record) throws IOException, WriteProcessException {
+ String tableName = record.getTableName();
+ if (tableName == null) {
+ tableName = this.tableName;
+ }
+ if (tableName == null) {
+ throw new WriteProcessException("Table name is null");
+ }
+
+ final TableSchema tableSchema = getSchema().getTableSchemaMap().get(tableName);
+ if (tableSchema == null) {
+ throw new NoTableException(tableName);
+ }
+
+ IDeviceID deviceId = record.getDeviceId(tableSchema);
+ List<DataPoint> fieldDataPoints = new ArrayList<>(record.dataPointList.size());
+ for (DataPoint dataPoint : record.dataPointList) {
+ int columnIndex = tableSchema.findColumnIndex(dataPoint.getMeasurementId());
+ if (columnIndex < 0) {
+ throw new NoMeasurementException(dataPoint.getMeasurementId());
+ }
+ ColumnCategory columnCategory = tableSchema.getColumnTypes().get(columnIndex);
+ if (columnCategory == ColumnCategory.FIELD) {
+ fieldDataPoints.add(dataPoint);
+ dataPoint.setMeasurementSchema(tableSchema.getColumnSchemas().get(columnIndex));
+ }
+ }
+ recordCount +=
+ tryToInitialGroupWriter(deviceId, isTableWriteAligned, true)
+ .write(record.time, fieldDataPoints);
+ checkMemorySizeAndMayFlushChunks();
+ }
+
@Override
protected void initAllSeriesWriterForAlignedSeries(
AlignedChunkGroupWriterImpl alignedChunkGroupWriter) throws IOException {
@@ -90,6 +127,9 @@ public class DeviceTableModelWriter extends AbstractTableModelTsFileWriter {
}
tablet.setTableName(this.tableName);
final TableSchema tableSchema = getSchema().getTableSchemaMap().get(tableName);
+ if (tableSchema == null) {
+ throw new NoTableException(tabletTableName);
+ }
List<ColumnCategory> columnCategoryListForTablet = new ArrayList<>(tablet.getSchemas().size());
for (IMeasurementSchema writingColumnSchema : tablet.getSchemas()) {
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/ITsFileWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/ITsFileWriter.java
index 8610ee1f..25ccb031 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/ITsFileWriter.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/ITsFileWriter.java
@@ -21,6 +21,7 @@ package org.apache.tsfile.write.v4;
import org.apache.tsfile.annotations.TsFileApi;
import org.apache.tsfile.exception.write.WriteProcessException;
+import org.apache.tsfile.write.record.TSRecord;
import org.apache.tsfile.write.record.Tablet;
import java.io.IOException;
@@ -32,4 +33,7 @@ public interface ITsFileWriter extends AutoCloseable {
@TsFileApi
void close();
+
+ @TsFileApi
+ void write(TSRecord record) throws IOException, WriteProcessException;
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/TsFileTreeWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/TsFileTreeWriter.java
new file mode 100644
index 00000000..4bd7649d
--- /dev/null
+++ b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/TsFileTreeWriter.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tsfile.write.v4;
+
+import org.apache.tsfile.exception.write.WriteProcessException;
+import org.apache.tsfile.write.TsFileWriter;
+import org.apache.tsfile.write.record.TSRecord;
+import org.apache.tsfile.write.record.Tablet;
+import org.apache.tsfile.write.schema.IMeasurementSchema;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * TsFileTreeWriter provides a tree-based interface for writing data to TsFiles. It simplifies the
+ * writing process by handling schema registration and data writing with a focus on tree model
+ * operations.
+ */
+public class TsFileTreeWriter implements AutoCloseable {
+
+ private final TsFileWriter tsFileWriter;
+
+ /**
+ * Constructs a TsFileTreeWriter with the specified TsFileWriter and memory threshold.
+ *
+ * @param file the output file
+ * @param memoryThreshold the memory threshold for flushing data
+ */
+ TsFileTreeWriter(File file, int memoryThreshold) throws IOException {
+ this.tsFileWriter = new TsFileWriter(file);
+ tsFileWriter.setMemoryThreshold(memoryThreshold);
+ }
+
+ /**
+ * Registers a non-aligned timeseries for the specified device.
+ *
+ * @param deviceId the device identifier
+ * @param schema the measurement schema to register
+ * @throws WriteProcessException if registration fails
+ */
+ public void registerTimeseries(String deviceId, IMeasurementSchema schema)
+ throws WriteProcessException {
+ try {
+ tsFileWriter.registerTimeseries(deviceId, schema);
+ } catch (WriteProcessException e) {
+ throw new WriteProcessException("Failed to register timeseries for device " + deviceId, e);
+ }
+ }
+
+ /**
+ * Registers a group of aligned timeseries for the specified device.
+ *
+ * @param deviceId the device identifier
+ * @param schemas list of measurement schemas to register as aligned
+ * @throws WriteProcessException if registration fails
+ */
+ public void registerAlignedTimeseries(String deviceId, List<IMeasurementSchema> schemas)
+ throws WriteProcessException {
+ try {
+ tsFileWriter.registerAlignedTimeseries(deviceId, schemas);
+ } catch (WriteProcessException e) {
+ throw new WriteProcessException(
+ "Failed to register aligned timeseries for device " + deviceId, e);
+ }
+ }
+
+ /**
+ * Writes a Tablet (batch of data points) to the TsFile.
+ *
+ * @param tablet the tablet containing data points to write
+ * @throws IOException if an I/O error occurs
+ * @throws WriteProcessException if writing fails
+ */
+ public void write(Tablet tablet) throws IOException, WriteProcessException {
+ try {
+ tsFileWriter.writeTree(tablet);
+ } catch (IOException | WriteProcessException e) {
+ throw new WriteProcessException("Failed to write tablet data", e);
+ }
+ }
+
+ /**
+ * Writes a TSRecord (single record) to the TsFile.
+ *
+ * @param record the TSRecord to write
+ * @throws IOException if an I/O error occurs
+ * @throws WriteProcessException if writing fails
+ */
+ public void write(TSRecord record) throws IOException, WriteProcessException {
+ try {
+ tsFileWriter.writeRecord(record);
+ } catch (IOException | WriteProcessException e) {
+ throw new WriteProcessException("Failed to write TSRecord", e);
+ }
+ }
+
+ /**
+ * Flushes any buffered data and closes the writer. After closing, the writer cannot be used
+ * anymore.
+ *
+ * @throws IOException if an I/O error occurs during closing
+ */
+ @Override
+ public void close() throws IOException {
+ try {
+ tsFileWriter.close();
+ } catch (IOException e) {
+ throw new IOException("Failed to close TsFileTreeWriter", e);
+ }
+ }
+}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/TsFileTreeWriterBuilder.java b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/TsFileTreeWriterBuilder.java
new file mode 100644
index 00000000..453ff8f5
--- /dev/null
+++ b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/TsFileTreeWriterBuilder.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tsfile.write.v4;
+
+import org.apache.tsfile.common.conf.TSFileDescriptor;
+
+import java.io.File;
+import java.io.IOException;
+
+/** Builder class for TsFileTreeWriter to provide a fluent interface for configuration. */
+public class TsFileTreeWriterBuilder {
+
+ private File file;
+ private int memoryThreshold = TSFileDescriptor.getInstance().getConfig().getGroupSizeInByte();
+
+ /**
+ * Sets the output file for the TsFileTreeWriter.
+ *
+ * @param file the output file
+ * @return this builder instance
+ */
+ public TsFileTreeWriterBuilder file(File file) {
+ this.file = file;
+ return this;
+ }
+
+ /**
+ * Sets the memory threshold for flushing data.
+ *
+ * @param memoryThreshold the threshold in bytes
+ * @return this builder instance
+ */
+ public TsFileTreeWriterBuilder memoryThreshold(int memoryThreshold) {
+ this.memoryThreshold = memoryThreshold;
+ return this;
+ }
+
+ /**
+ * Builds and returns a new TsFileTreeWriter instance with the configured settings.
+ *
+ * @return a new TsFileTreeWriter instance
+ * @throws IOException if the TsFileWriter cannot be initialized
+ */
+ public TsFileTreeWriter build() throws IOException {
+ if (file == null) {
+ throw new IllegalArgumentException("Output file must be specified");
+ }
+
+ return new TsFileTreeWriter(file, memoryThreshold);
+ }
+}
diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileTreeReaderTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileTreeReaderTest.java
new file mode 100644
index 00000000..21d953dc
--- /dev/null
+++ b/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileTreeReaderTest.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tsfile.read;
+
+import org.apache.tsfile.common.conf.TSFileConfig;
+import org.apache.tsfile.common.conf.TSFileDescriptor;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.exception.write.WriteProcessException;
+import org.apache.tsfile.read.query.dataset.ResultSet;
+import org.apache.tsfile.read.query.dataset.ResultSetMetadata;
+import org.apache.tsfile.read.v4.ITsFileTreeReader;
+import org.apache.tsfile.read.v4.TsFileTreeReaderBuilder;
+import org.apache.tsfile.write.record.TSRecord;
+import org.apache.tsfile.write.record.Tablet;
+import org.apache.tsfile.write.schema.IMeasurementSchema;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+import org.apache.tsfile.write.v4.TsFileTreeWriter;
+import org.apache.tsfile.write.v4.TsFileTreeWriterBuilder;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+import java.util.stream.Collectors;
+
+public class TsFileTreeReaderTest {
+
+ private static final String FILE_PATH = "test_v4_tree_reader.tsfile";
+ private ITsFileTreeReader treeReader;
+
+ @Before
+ public void setUp() throws IOException {
+ generateTestTsFile();
+ treeReader = new TsFileTreeReaderBuilder().file(new File(FILE_PATH)).build();
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ if (treeReader != null) {
+ treeReader.close();
+ }
+ new File(FILE_PATH).delete();
+ }
+
+ // Test basic query functionality with device IDs and measurements
+ @Test
+ public void testBasicQuery() throws IOException {
+ List<String> deviceIds = Arrays.asList("d1", "d2");
+ List<String> measurements = Arrays.asList("s1", "s2");
+ long startTime = Long.MIN_VALUE;
+ long endTime = Long.MAX_VALUE;
+
+ ResultSet resultSet = treeReader.query(deviceIds, measurements, startTime, endTime);
+
+ int countVal = 0;
+ while (resultSet.next()) {
+ Assert.assertEquals(resultSet.getLong(1), countVal);
+ Assert.assertEquals(resultSet.getInt(2), countVal);
+ Assert.assertEquals(resultSet.getLong(3), countVal);
+ Assert.assertEquals(resultSet.getInt(4), countVal);
+ countVal++;
+ }
+ Assert.assertTrue("Should return at least one record", countVal > 0);
+ }
+
+ // Test iterator-based query functionality
+ @Test
+ public void testIteratorQuery() throws IOException {
+ List<String> deviceIds = Arrays.asList("d2", "d1");
+ List<String> measurements = Arrays.asList("s2", "s1");
+ long startTime = Long.MIN_VALUE;
+ long endTime = Long.MAX_VALUE;
+
+ ResultSet resultSet = treeReader.query(deviceIds, measurements, startTime, endTime);
+ Iterator<TSRecord> recordIterator = resultSet.recordIterator();
+
+ int countVal = 0;
+ while (recordIterator.hasNext()) {
+ TSRecord record = recordIterator.next();
+ if (record.deviceId.toString().equals("d1")) {
+ Assert.assertEquals("s2", record.dataPointList.get(1).getMeasurementId());
+ Assert.assertEquals("s1", record.dataPointList.get(2).getMeasurementId());
+ } else if (record.deviceId.toString().equals("d2")) {
+ Assert.assertEquals("s1", record.dataPointList.get(2).getMeasurementId());
+ Assert.assertNull(record.dataPointList.get(1));
+ } else {
+ Assert.fail("Unexpected deviceId: " + record.deviceId.toString());
+ }
+ countVal++;
+ }
+ Assert.assertTrue("Should return at least one record", countVal > 0);
+ }
+
+ // Test retrieving all device IDs from the TsFile
+ @Test
+ public void testGetAllDeviceIds() throws IOException {
+ List<String> deviceIds = treeReader.getAllDeviceIds();
+ Assert.assertEquals(2, deviceIds.size());
+ Assert.assertTrue(deviceIds.contains("d1"));
+ Assert.assertTrue(deviceIds.contains("d2"));
+ }
+
+ // Test ResultSet metadata functionality
+ @Test
+ public void testResultSetMetadata() throws IOException {
+ List<String> deviceIds = Arrays.asList("d1", "d2");
+ List<String> measurements = Arrays.asList("s1", "s2");
+ long startTime = Long.MIN_VALUE;
+ long endTime = Long.MAX_VALUE;
+
+ ResultSet resultSet = treeReader.query(deviceIds, measurements, startTime, endTime);
+ ResultSetMetadata resultSetMetadata = resultSet.getMetadata();
+ Assert.assertEquals("Time", resultSetMetadata.getColumnName(1));
+ Assert.assertEquals("d1.s1", resultSetMetadata.getColumnName(2));
+ Assert.assertEquals("d1.s2", resultSetMetadata.getColumnName(3));
+ Assert.assertEquals("d2.s1", resultSetMetadata.getColumnName(4));
+ }
+
+ // Test retrieving schema information for a specific device
+ @Test
+ public void testGetDeviceSchema() throws IOException {
+ List<MeasurementSchema> schemas = treeReader.getDeviceSchema("d1");
+ List<String> measurementNames =
+ schemas.stream().map(MeasurementSchema::getMeasurementName).collect(Collectors.toList());
+
+ Assert.assertTrue(measurementNames.contains("s1"));
+ Assert.assertTrue(measurementNames.contains("s2"));
+ }
+
+ // Helper method to generate test TsFile data
+ private void generateTestTsFile() throws IOException {
+ TSFileConfig config = TSFileDescriptor.getInstance().getConfig();
+ File file = new File(TsFileTreeReaderTest.FILE_PATH);
+
+ try (TsFileTreeWriter writer = new TsFileTreeWriterBuilder().file(file).build()) {
+ // Register time series schemas
+ IMeasurementSchema schema1 = new MeasurementSchema("s1", TSDataType.INT32);
+ IMeasurementSchema schema2 = new MeasurementSchema("s2", TSDataType.INT64);
+
+ // Schema for device d1
+ writer.registerTimeseries("d1", schema1);
+ writer.registerTimeseries("d1", schema2);
+
+ // Schema for device d2
+ writer.registerTimeseries("d2", schema1);
+
+ // Write data for device d1 using Tablet
+ Tablet d1Tablet = new Tablet("d1", Arrays.asList(schema1, schema2));
+ for (int i = 0; i < 10; i++) {
+ d1Tablet.addTimestamp(i, i);
+ d1Tablet.addValue(i, 0, i);
+ d1Tablet.addValue(i, 1, (long) i);
+ }
+ writer.write(d1Tablet);
+
+ // Write data for device d2 using TSRecord
+ TSRecord record;
+ for (int i = 0; i < 10; i++) {
+ record = new TSRecord("d2", i);
+ record.addPoint("s1", i);
+ writer.write(record);
+ }
+ } catch (WriteProcessException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/query/ResultSetTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/query/ResultSetTest.java
index a9f8766b..b24e6e6d 100644
--- a/java/tsfile/src/test/java/org/apache/tsfile/read/query/ResultSetTest.java
+++ b/java/tsfile/src/test/java/org/apache/tsfile/read/query/ResultSetTest.java
@@ -22,10 +22,12 @@ package org.apache.tsfile.read.query;
import org.apache.tsfile.enums.ColumnCategory;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.metadata.TableSchema;
+import org.apache.tsfile.read.filter.factory.TagFilterBuilder;
import org.apache.tsfile.read.query.dataset.ResultSet;
import org.apache.tsfile.read.query.dataset.ResultSetMetadata;
import org.apache.tsfile.read.v4.DeviceTableModelReader;
import org.apache.tsfile.utils.TsFileGeneratorForTest;
+import org.apache.tsfile.write.record.TSRecord;
import org.apache.tsfile.write.record.Tablet;
import org.apache.tsfile.write.schema.MeasurementSchema;
import org.apache.tsfile.write.v4.ITsFileWriter;
@@ -40,6 +42,7 @@ import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.util.Arrays;
+import java.util.Iterator;
public class ResultSetTest {
@@ -139,6 +142,372 @@ public class ResultSetTest {
}
}
+ @Test
+ public void testQueryTableWithAllTagFilters() throws Exception {
+ TableSchema tableSchema =
+ new TableSchema(
+ "t1",
+ Arrays.asList(
+ new MeasurementSchema("id1", TSDataType.STRING),
+ new MeasurementSchema("id2", TSDataType.STRING),
+ new MeasurementSchema("s1", TSDataType.BOOLEAN),
+ new MeasurementSchema("s2", TSDataType.BOOLEAN)),
+ Arrays.asList(
+ ColumnCategory.TAG,
+ ColumnCategory.TAG,
+ ColumnCategory.FIELD,
+ ColumnCategory.FIELD));
+
+ Tablet tablet =
+ new Tablet(
+ Arrays.asList("id1", "id2", "s1", "s2"),
+ Arrays.asList(
+ TSDataType.STRING, TSDataType.STRING, TSDataType.BOOLEAN, TSDataType.BOOLEAN),
+ 1024);
+
+ // Row 0
+ tablet.addTimestamp(0, 0);
+ tablet.addValue("id1", 0, "id_01");
+ tablet.addValue("id2", 0, "name_01");
+ tablet.addValue("s1", 0, true);
+ tablet.addValue("s2", 0, false);
+
+ // Row 1
+ tablet.addTimestamp(1, 1);
+ tablet.addValue("id1", 1, "id_02");
+ tablet.addValue("id2", 1, "name_02");
+ tablet.addValue("s1", 1, false);
+ tablet.addValue("s2", 1, true);
+
+ // Row 2
+ tablet.addTimestamp(2, 2);
+ tablet.addValue("id1", 2, "id_03");
+ tablet.addValue("id2", 2, "name_03");
+ tablet.addValue("s1", 2, true);
+ tablet.addValue("s2", 2, true);
+
+ // Row 3
+ tablet.addTimestamp(3, 3);
+ tablet.addValue("id1", 3, "id_04");
+ tablet.addValue("id2", 3, "name_04");
+ tablet.addValue("s1", 3, false);
+ tablet.addValue("s2", 3, false);
+
+ try (ITsFileWriter writer =
+ new TsFileWriterBuilder().file(tsfile).tableSchema(tableSchema).build()) {
+ writer.write(tablet);
+ }
+
+ TagFilterBuilder filterBuilder = new TagFilterBuilder(tableSchema);
+
+ // eq
+ try (DeviceTableModelReader r = new DeviceTableModelReader(tsfile);
+ ResultSet rs =
+ r.query(
+ "T1",
+ Arrays.asList("ID1", "ID2", "S2", "S1"),
+ 0,
+ 3,
+ filterBuilder.eq("ID1", "id_02"))) {
+ Assert.assertTrue(rs.next());
+ Assert.assertEquals(1, rs.getLong(1));
+ Assert.assertEquals("id_02", rs.getString(2));
+ Assert.assertEquals("name_02", rs.getString(3));
+ }
+
+ // neq
+ try (DeviceTableModelReader r = new DeviceTableModelReader(tsfile);
+ ResultSet rs =
+ r.query(
+ "T1",
+ Arrays.asList("ID1", "ID2", "S2", "S1"),
+ 0,
+ 3,
+ filterBuilder.neq("ID1", "id_02"))) {
+ Assert.assertTrue(rs.next());
+ Assert.assertEquals(0, rs.getLong(1));
+ Assert.assertTrue(rs.next());
+ Assert.assertEquals(2, rs.getLong(1));
+ Assert.assertTrue(rs.next());
+ Assert.assertEquals(3, rs.getLong(1));
+ }
+
+ // lt
+ try (DeviceTableModelReader r = new DeviceTableModelReader(tsfile);
+ ResultSet rs =
+ r.query(
+ "T1",
+ Arrays.asList("ID1", "ID2", "S2", "S1"),
+ 0,
+ 3,
+ filterBuilder.lt("ID1", "id_03"))) {
+ Assert.assertTrue(rs.next());
+ Assert.assertEquals(0, rs.getLong(1));
+ Assert.assertTrue(rs.next());
+ Assert.assertEquals(1, rs.getLong(1));
+ }
+
+ // lteq
+ try (DeviceTableModelReader r = new DeviceTableModelReader(tsfile);
+ ResultSet rs =
+ r.query(
+ "T1",
+ Arrays.asList("ID1", "ID2", "S2", "S1"),
+ 0,
+ 3,
+ filterBuilder.lteq("ID1", "id_03"))) {
+ Assert.assertTrue(rs.next());
+ Assert.assertEquals(0, rs.getLong(1));
+ Assert.assertTrue(rs.next());
+ Assert.assertEquals(1, rs.getLong(1));
+ Assert.assertTrue(rs.next());
+ Assert.assertEquals(2, rs.getLong(1));
+ }
+
+ // gt
+ try (DeviceTableModelReader r = new DeviceTableModelReader(tsfile);
+ ResultSet rs =
+ r.query(
+ "T1",
+ Arrays.asList("ID1", "ID2", "S2", "S1"),
+ 0,
+ 3,
+ filterBuilder.gt("ID1", "id_02"))) {
+ Assert.assertTrue(rs.next());
+ Assert.assertEquals(2, rs.getLong(1));
+ Assert.assertTrue(rs.next());
+ Assert.assertEquals(3, rs.getLong(1));
+ }
+
+ // gteq
+ try (DeviceTableModelReader r = new DeviceTableModelReader(tsfile);
+ ResultSet rs =
+ r.query(
+ "T1",
+ Arrays.asList("ID1", "ID2", "S2", "S1"),
+ 0,
+ 3,
+ filterBuilder.gteq("ID1", "id_02"))) {
+ Assert.assertTrue(rs.next());
+ Assert.assertEquals(1, rs.getLong(1));
+ Assert.assertTrue(rs.next());
+ Assert.assertEquals(2, rs.getLong(1));
+ Assert.assertTrue(rs.next());
+ Assert.assertEquals(3, rs.getLong(1));
+ }
+
+ // betweenAnd
+ try (DeviceTableModelReader r = new DeviceTableModelReader(tsfile);
+ ResultSet rs =
+ r.query(
+ "T1",
+ Arrays.asList("ID1", "ID2", "S2", "S1"),
+ 0,
+ 3,
+ filterBuilder.betweenAnd("ID1", "id_02", "id_03"))) {
+ Assert.assertTrue(rs.next());
+ Assert.assertEquals(1, rs.getLong(1)); // id_02
+ Assert.assertTrue(rs.next());
+ Assert.assertEquals(2, rs.getLong(1)); // id_03
+ }
+
+ // notBetweenAnd
+ try (DeviceTableModelReader r = new DeviceTableModelReader(tsfile);
+ ResultSet rs =
+ r.query(
+ "T1",
+ Arrays.asList("ID1", "ID2", "S2", "S1"),
+ 0,
+ 3,
+ filterBuilder.notBetweenAnd("ID1", "id_02", "id_03"))) {
+ Assert.assertTrue(rs.next());
+ Assert.assertEquals(0, rs.getLong(1)); // id_01
+ Assert.assertTrue(rs.next());
+ Assert.assertEquals(3, rs.getLong(1)); // id_04
+ }
+
+ // and
+ try (DeviceTableModelReader r = new DeviceTableModelReader(tsfile);
+ ResultSet rs =
+ r.query(
+ "T1",
+ Arrays.asList("ID1", "ID2", "S2", "S1"),
+ 0,
+ 3,
+ filterBuilder.and(
+ filterBuilder.gteq("ID1", "id_02"), filterBuilder.lteq("ID1", "id_03")))) {
+ Assert.assertTrue(rs.next());
+ Assert.assertEquals(1, rs.getLong(1)); // id_02
+ Assert.assertTrue(rs.next());
+ Assert.assertEquals(2, rs.getLong(1)); // id_03
+ }
+
+ // or
+ try (DeviceTableModelReader r = new DeviceTableModelReader(tsfile);
+ ResultSet rs =
+ r.query(
+ "T1",
+ Arrays.asList("ID1", "ID2", "S2", "S1"),
+ 0,
+ 3,
+ filterBuilder.or(
+ filterBuilder.eq("ID1", "id_01"), filterBuilder.eq("ID1", "id_04")))) {
+ Assert.assertTrue(rs.next());
+ Assert.assertEquals(0, rs.getLong(1)); // id_01
+ Assert.assertTrue(rs.next());
+ Assert.assertEquals(3, rs.getLong(1)); // id_04
+ }
+
+ // not
+ try (DeviceTableModelReader r = new DeviceTableModelReader(tsfile);
+ ResultSet rs =
+ r.query(
+ "T1",
+ Arrays.asList("ID1", "ID2", "S2", "S1"),
+ 0,
+ 3,
+ filterBuilder.not(filterBuilder.eq("ID1", "id_02")))) {
+ Assert.assertTrue(rs.next());
+ Assert.assertEquals(0, rs.getLong(1)); // id_01
+ Assert.assertTrue(rs.next());
+ Assert.assertEquals(2, rs.getLong(1)); // id_03
+ Assert.assertTrue(rs.next());
+ Assert.assertEquals(3, rs.getLong(1)); // id_04
+ }
+
+ // regExp
+ try (DeviceTableModelReader r = new DeviceTableModelReader(tsfile);
+ ResultSet rs =
+ r.query(
+ "T1",
+ Arrays.asList("ID1", "ID2", "S2", "S1"),
+ 0,
+ 3,
+ filterBuilder.regExp("ID1", "id_0[23]"))) {
+ Assert.assertTrue(rs.next());
+ Assert.assertEquals(1, rs.getLong(1)); // id_02
+ Assert.assertTrue(rs.next());
+ Assert.assertEquals(2, rs.getLong(1)); // id_03
+ }
+
+ // notRegExp
+ try (DeviceTableModelReader r = new DeviceTableModelReader(tsfile);
+ ResultSet rs =
+ r.query(
+ "T1",
+ Arrays.asList("ID1", "ID2", "S2", "S1"),
+ 0,
+ 3,
+ filterBuilder.notRegExp("ID1", "id_0[23]"))) {
+ Assert.assertTrue(rs.next());
+ Assert.assertEquals(0, rs.getLong(1)); // id_01
+ Assert.assertTrue(rs.next());
+ Assert.assertEquals(3, rs.getLong(1)); // id_04
+ }
+
+ // like
+ try (DeviceTableModelReader r = new DeviceTableModelReader(tsfile);
+ ResultSet rs =
+ r.query(
+ "T1",
+ Arrays.asList("ID1", "ID2", "S2", "S1"),
+ 0,
+ 3,
+ filterBuilder.like("ID1", "id_0_"))) {
+ Assert.assertTrue(rs.next());
+ Assert.assertEquals(0, rs.getLong(1)); // id_01
+ Assert.assertTrue(rs.next());
+ Assert.assertEquals(1, rs.getLong(1)); // id_02
+ Assert.assertTrue(rs.next());
+ Assert.assertEquals(2, rs.getLong(1)); // id_03
+ Assert.assertTrue(rs.next());
+ Assert.assertEquals(3, rs.getLong(1)); // id_04
+ }
+
+ // notLike
+ try (DeviceTableModelReader r = new DeviceTableModelReader(tsfile);
+ ResultSet rs =
+ r.query(
+ "T1",
+ Arrays.asList("ID1", "ID2", "S2", "S1"),
+ 0,
+ 3,
+ filterBuilder.notLike("ID1", "id_0_"))) {
+ Assert.assertFalse(rs.next());
+ }
+ }
+
+ @Test
+ public void testQueryTableByIterator() throws Exception {
+ TableSchema tableSchema =
+ new TableSchema(
+ "t1",
+ Arrays.asList(
+ new MeasurementSchema("id1", TSDataType.STRING),
+ new MeasurementSchema("id2", TSDataType.STRING),
+ new MeasurementSchema("s1", TSDataType.BOOLEAN),
+ new MeasurementSchema("s2", TSDataType.BOOLEAN)),
+ Arrays.asList(
+ ColumnCategory.TAG,
+ ColumnCategory.TAG,
+ ColumnCategory.FIELD,
+ ColumnCategory.FIELD));
+ Tablet tablet =
+ new Tablet(
+ Arrays.asList("id1", "id2", "s1", "s2"),
+ Arrays.asList(
+ TSDataType.STRING, TSDataType.STRING, TSDataType.BOOLEAN, TSDataType.BOOLEAN),
+ 1024);
+ tablet.addTimestamp(0, 0);
+ tablet.addValue("id1", 0, "id_field1");
+ tablet.addValue("id2", 0, "id_field2");
+ tablet.addValue("s1", 0, true);
+ tablet.addValue("s2", 0, false);
+
+ tablet.addTimestamp(1, 1);
+ tablet.addValue("id1", 1, "id_field1_2");
+ tablet.addValue("s2", 1, true);
+
+ tablet.addTimestamp(2, 2);
+
+ try (ITsFileWriter writer =
+ new TsFileWriterBuilder().file(tsfile).tableSchema(tableSchema).build()) {
+ writer.write(tablet);
+ }
+
+ try (DeviceTableModelReader tsFileReader = new DeviceTableModelReader(tsfile);
+ ResultSet resultSet =
+ tsFileReader.query("T1", Arrays.asList("ID1", "ID2", "S2", "S1"), 0, 2); ) {
+ Iterator<TSRecord> tsRecordIterator = resultSet.recordIterator();
+
+ Assert.assertTrue(tsRecordIterator.hasNext());
+ TSRecord tsRecord = tsRecordIterator.next();
+ Assert.assertEquals(2, tsRecord.time);
+ Assert.assertNull(tsRecord.dataPointList.get(0));
+ Assert.assertNull(tsRecord.dataPointList.get(1));
+ Assert.assertNull(tsRecord.dataPointList.get(2));
+ Assert.assertNull(tsRecord.dataPointList.get(3));
+
+ Assert.assertTrue(tsRecordIterator.hasNext());
+ tsRecord = tsRecordIterator.next();
+ Assert.assertEquals(0, tsRecord.time);
+ Assert.assertEquals("id_field1", tsRecord.dataPointList.get(0).getValue().toString());
+ Assert.assertEquals("id_field2", tsRecord.dataPointList.get(1).getValue().toString());
+ Assert.assertFalse((Boolean) tsRecord.dataPointList.get(2).getValue());
+ Assert.assertTrue((Boolean) tsRecord.dataPointList.get(3).getValue());
+
+ Assert.assertTrue(tsRecordIterator.hasNext());
+ tsRecord = tsRecordIterator.next();
+ Assert.assertEquals(1, tsRecord.time);
+ Assert.assertEquals("id_field1_2", tsRecord.dataPointList.get(0).getValue().toString());
+ Assert.assertNull(tsRecord.dataPointList.get(1));
+ Assert.assertTrue((Boolean) tsRecord.dataPointList.get(2).getValue());
+ Assert.assertNull(tsRecord.dataPointList.get(3));
+
+ Assert.assertFalse(tsRecordIterator.hasNext());
+ }
+ }
+
@Test
public void testQueryWithMaxValue() throws Exception {
TableSchema tableSchema =
diff --git a/java/tsfile/src/test/java/org/apache/tsfile/tableview/PerformanceTest.java b/java/tsfile/src/test/java/org/apache/tsfile/tableview/PerformanceTest.java
index fdcf8ca6..6b49ab81 100644
--- a/java/tsfile/src/test/java/org/apache/tsfile/tableview/PerformanceTest.java
+++ b/java/tsfile/src/test/java/org/apache/tsfile/tableview/PerformanceTest.java
@@ -24,28 +24,23 @@ import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.exception.read.ReadProcessException;
import org.apache.tsfile.exception.write.WriteProcessException;
import org.apache.tsfile.file.metadata.IDeviceID;
-import org.apache.tsfile.file.metadata.IDeviceID.Factory;
+import org.apache.tsfile.file.metadata.PlainDeviceID;
import org.apache.tsfile.file.metadata.StringArrayDeviceID;
import org.apache.tsfile.file.metadata.TableSchema;
import org.apache.tsfile.file.metadata.enums.CompressionType;
import org.apache.tsfile.file.metadata.enums.TSEncoding;
-import org.apache.tsfile.read.TsFileSequenceReader;
-import org.apache.tsfile.read.common.Path;
-import org.apache.tsfile.read.common.block.TsBlock;
-import org.apache.tsfile.read.controller.CachedChunkLoaderImpl;
-import org.apache.tsfile.read.controller.MetadataQuerierByFileImpl;
-import org.apache.tsfile.read.expression.QueryExpression;
-import org.apache.tsfile.read.query.dataset.QueryDataSet;
-import org.apache.tsfile.read.query.executor.QueryExecutor;
-import org.apache.tsfile.read.query.executor.TableQueryExecutor;
-import org.apache.tsfile.read.query.executor.TableQueryExecutor.TableQueryOrdering;
-import org.apache.tsfile.read.query.executor.TsFileExecutor;
-import org.apache.tsfile.read.reader.block.TsBlockReader;
+import org.apache.tsfile.read.query.dataset.ResultSet;
+import org.apache.tsfile.read.v4.ITsFileReader;
+import org.apache.tsfile.read.v4.ITsFileTreeReader;
+import org.apache.tsfile.read.v4.TsFileReaderBuilder;
+import org.apache.tsfile.read.v4.TsFileTreeReaderBuilder;
import org.apache.tsfile.utils.Pair;
import org.apache.tsfile.write.TsFileWriter;
import org.apache.tsfile.write.record.Tablet;
import org.apache.tsfile.write.schema.IMeasurementSchema;
import org.apache.tsfile.write.schema.MeasurementSchema;
+import org.apache.tsfile.write.v4.TsFileTreeWriter;
+import org.apache.tsfile.write.v4.TsFileTreeWriterBuilder;
import java.io.File;
import java.io.IOException;
@@ -54,140 +49,252 @@ import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
-import static org.junit.Assert.assertTrue;
-
+/** Performance test comparing TreeV4 and Table modes in TsFile with microsecond precision. */
public class PerformanceTest {
+ // Test configuration constants - default values set to 10
private final String testDir = "target" + File.separator + "tableViewTest";
- private final int idSchemaCnt = 3;
- private final int measurementSchemaCnt = 100;
- private final int tableCnt = 100;
- private final int devicePerTable = 10;
- private final int pointPerSeries = 100;
- private final int tabletCnt = 100;
+ private int measurementSchemaCnt = 100; // Number of measurement schemas (data points) per device
+ private int tableCnt = 10; // Number of tables in the database
+ private int devicePerTable = 10; // Number of devices allocated per table
+ private int tabletCnt = 10; // Number of tablets each device writes to
+ private int pointPerSeries = 10; // Number of data points per time series within a single tablet
+ private final int idSchemaCnt = 3; // Number of ID schemas
private List<IMeasurementSchema> idSchemas;
private List<IMeasurementSchema> measurementSchemas;
- private List<Long> registerTimeList = new ArrayList<>();
- private List<Long> writeTimeList = new ArrayList<>();
- private List<Long> closeTimeList = new ArrayList<>();
- private List<Long> queryTimeList = new ArrayList<>();
- private List<Long> fileSizeList = new ArrayList<>();
-
public static void main(String[] args) throws Exception {
final PerformanceTest test = new PerformanceTest();
test.initSchemas();
- int repetitionCnt = 10;
- for (int i = 0; i < repetitionCnt; i++) {
- test.testTable();
- // test.testTree();
+ // Test different variable values
+ int[] testValues = {10, 20, 30, 40, 50};
+
+ System.out.println("=== Controlled Variable Performance Test (μs precision) ===");
+
+ // Test measurementSchemaCnt impact
+ System.out.println(
+ "\n--- Testing measurementSchemaCnt impact (tableCnt=10, devicePerTable=10, pointPerSeries=10, tabletCnt=10) ---");
+ for (int value : testValues) {
+ test.measurementSchemaCnt = value;
+ test.tableCnt = 10;
+ test.devicePerTable = 10;
+ test.pointPerSeries = 10;
+ test.tabletCnt = 10;
+ test.initSchemas(); // Reinitialize schemas with new measurement count
+ test.runSingleTest("measurementSchemaCnt=" + value);
+ }
+
+ // Test tableCnt impact
+ System.out.println(
+ "\n--- Testing tableCnt impact (measurementSchemaCnt=100, devicePerTable=10, pointPerSeries=10, tabletCnt=10) ---");
+ for (int value : testValues) {
+ test.measurementSchemaCnt = 100;
+ test.tableCnt = value;
+ test.devicePerTable = 10;
+ test.pointPerSeries = 10;
+ test.tabletCnt = 10;
+ test.initSchemas();
+ test.runSingleTest("tableCnt=" + value);
+ }
+
+ // Test devicePerTable impact
+ System.out.println(
+ "\n--- Testing devicePerTable impact (measurementSchemaCnt=100, tableCnt=10, pointPerSeries=10, tabletCnt=10) ---");
+ for (int value : testValues) {
+ test.measurementSchemaCnt = 100;
+ test.tableCnt = 10;
+ test.devicePerTable = value;
+ test.pointPerSeries = 10;
+ test.tabletCnt = 10;
+ test.initSchemas();
+ test.runSingleTest("devicePerTable=" + value);
+ }
+
+ // Test pointPerSeries impact
+ System.out.println(
+ "\n--- Testing pointPerSeries impact (measurementSchemaCnt=100, tableCnt=10, devicePerTable=10, tabletCnt=10) ---");
+ for (int value : testValues) {
+ test.measurementSchemaCnt = 100;
+ test.tableCnt = 10;
+ test.devicePerTable = 10;
+ test.pointPerSeries = value;
+ test.tabletCnt = 10;
+ test.initSchemas();
+ test.runSingleTest("pointPerSeries=" + value);
}
- final double registerTime =
- test.registerTimeList.subList(repetitionCnt / 2, repetitionCnt).stream()
- .mapToLong(l -> l)
- .average()
- .orElse(0.0f);
- final double writeTime =
- test.writeTimeList.subList(repetitionCnt / 2, repetitionCnt).stream()
- .mapToLong(l -> l)
- .average()
- .orElse(0.0f);
- final double closeTime =
- test.closeTimeList.subList(repetitionCnt / 2, repetitionCnt).stream()
- .mapToLong(l -> l)
- .average()
- .orElse(0.0f);
- final double queryTime =
- test.queryTimeList.subList(repetitionCnt / 2, repetitionCnt).stream()
- .mapToLong(l -> l)
- .average()
- .orElse(0.0f);
- final double fileSize =
- test.fileSizeList.subList(repetitionCnt / 2, repetitionCnt).stream()
- .mapToLong(l -> l)
- .average()
- .orElse(0.0f);
+ // Test tabletCnt impact
+ System.out.println(
+ "\n--- Testing tabletCnt impact (measurementSchemaCnt=100, tableCnt=10, devicePerTable=10, pointPerSeries=10) ---");
+ for (int value : testValues) {
+ test.measurementSchemaCnt = 100;
+ test.tableCnt = 10;
+ test.devicePerTable = 10;
+ test.pointPerSeries = 10;
+ test.tabletCnt = value;
+ test.initSchemas();
+ test.runSingleTest("tabletCnt=" + value);
+ }
+ }
+
+ /** Run single test and output results in microseconds */
+ private void runSingleTest(String testCase) throws Exception {
+ System.out.println("\nTest Configuration: " + testCase);
System.out.printf(
- "Register %fns, write %fns, close %fns, query %fns, fileSize %f %n",
- registerTime, writeTime, closeTime, queryTime, fileSize);
+ "measurementSchemaCnt=%d, tableCnt=%d, devicePerTable=%d, pointPerSeries=%d, tabletCnt=%d%n",
+ measurementSchemaCnt, tableCnt, devicePerTable, pointPerSeries, tabletCnt);
+
+ int repetitionCnt = 10;
+ int warmupRuns = repetitionCnt / 2;
+
+ // Test TreeV4 mode
+ testMode("TreeV4", repetitionCnt, warmupRuns);
+
+ // Test Table mode
+ testMode("Table", repetitionCnt, warmupRuns);
}
- private void initSchemas() {
- idSchemas = new ArrayList<>(idSchemaCnt);
- for (int i = 0; i < idSchemaCnt; i++) {
- idSchemas.add(
- new MeasurementSchema(
- "id" + i, TSDataType.TEXT, TSEncoding.PLAIN, CompressionType.UNCOMPRESSED));
- }
+ /** Tests a specific mode for multiple repetitions */
+ private void testMode(String mode, int totalRuns, int warmupRuns) throws Exception {
+ List<Long> registerTimeList = new ArrayList<>();
+ List<Long> writeTimeList = new ArrayList<>();
+ List<Long> closeTimeList = new ArrayList<>();
+ List<Long> queryTimeList = new ArrayList<>();
+ List<Long> fileSizeList = new ArrayList<>();
+
+ for (int i = 0; i < totalRuns; i++) {
+ switch (mode) {
+ case "TreeV4":
+ testTreeV4(registerTimeList, writeTimeList, closeTimeList, queryTimeList, fileSizeList);
+ break;
+ case "Table":
+ testTable(registerTimeList, writeTimeList, closeTimeList, queryTimeList, fileSizeList);
+ break;
+ }
- measurementSchemas = new ArrayList<>();
- for (int i = 0; i < measurementSchemaCnt; i++) {
- measurementSchemas.add(
- new MeasurementSchema(
- "s" + i, TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4));
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
}
+
+ calculateAndPrintStats(
+ mode,
+ registerTimeList,
+ writeTimeList,
+ closeTimeList,
+ queryTimeList,
+ fileSizeList,
+ warmupRuns,
+ totalRuns);
}
- private void testTree() throws IOException, WriteProcessException {
+ /** Calculates and prints performance statistics in microseconds */
+ private void calculateAndPrintStats(
+ String mode,
+ List<Long> registerList,
+ List<Long> writeList,
+ List<Long> closeList,
+ List<Long> queryList,
+ List<Long> sizeList,
+ int warmupRuns,
+ int totalRuns) {
+ List<Long> registerSub = registerList.subList(warmupRuns, totalRuns);
+ List<Long> writeSub = writeList.subList(warmupRuns, totalRuns);
+ List<Long> closeSub = closeList.subList(warmupRuns, totalRuns);
+ List<Long> querySub = queryList.subList(warmupRuns, totalRuns);
+ List<Long> sizeSub = sizeList.subList(warmupRuns, totalRuns);
+
+ final double registerTime = registerSub.stream().mapToLong(l -> l).average().orElse(0.0);
+ final double writeTime = writeSub.stream().mapToLong(l -> l).average().orElse(0.0);
+ final double closeTime = closeSub.stream().mapToLong(l -> l).average().orElse(0.0);
+ final double queryTime = querySub.stream().mapToLong(l -> l).average().orElse(0.0);
+ final double fileSize = sizeSub.stream().mapToLong(l -> l).average().orElse(0.0);
+
+ System.out.printf(
+ "[%s] Reg:%.1fμs Write:%.1fμs Close:%.1fμs Query:%.1fμs Size:%.1fKB%n",
+ mode,
+ registerTime / 1_000.0, // Convert ns to μs
+ writeTime / 1_000.0,
+ closeTime / 1_000.0,
+ queryTime / 1_000.0,
+ fileSize / 1024.0);
+ }
+
+ /** Tests the TreeV4 mode performance with microsecond timing */
+ private void testTreeV4(
+ List<Long> registerTimeList,
+ List<Long> writeTimeList,
+ List<Long> closeTimeList,
+ List<Long> queryTimeList,
+ List<Long> fileSizeList)
+ throws IOException {
long registerTimeSum = 0;
long writeTimeSum = 0;
long closeTimeSum = 0;
long queryTimeSum = 0;
long startTime;
- final File file = initFile();
- TsFileWriter tsFileWriter = new TsFileWriter(file);
- try {
+
+ final File file = initFile("test_v4_tree");
+ List<String> allDeviceIds = new ArrayList<>();
+
+ try (TsFileTreeWriter writer = new TsFileTreeWriterBuilder().file(file).build()) {
+ // Register schemas for all devices
startTime = System.nanoTime();
- registerTree(tsFileWriter);
+ for (int tableNum = 0; tableNum < tableCnt; tableNum++) {
+ for (int deviceNum = 0; deviceNum < devicePerTable; deviceNum++) {
+ String deviceId = genTreeDeviceId(tableNum, deviceNum).toString();
+ allDeviceIds.add(deviceId);
+ writer.registerAlignedTimeseries(deviceId, measurementSchemas);
+ }
+ }
registerTimeSum = System.nanoTime() - startTime;
- Tablet tablet = initTreeTablet();
+
+ // Write data tablets for all devices
+ Tablet tablet = new Tablet(null, measurementSchemas, pointPerSeries);
for (int tableNum = 0; tableNum < tableCnt; tableNum++) {
for (int deviceNum = 0; deviceNum < devicePerTable; deviceNum++) {
for (int tabletNum = 0; tabletNum < tabletCnt; tabletNum++) {
fillTreeTablet(tablet, tableNum, deviceNum, tabletNum);
startTime = System.nanoTime();
- tsFileWriter.writeAligned(tablet);
+ writer.write(tablet);
writeTimeSum += System.nanoTime() - startTime;
}
}
}
- } finally {
+
+ // Measure close time
startTime = System.nanoTime();
- tsFileWriter.close();
- closeTimeSum = System.nanoTime() - startTime;
+ } catch (WriteProcessException e) {
+ throw new RuntimeException(e);
}
+ closeTimeSum = System.nanoTime() - startTime;
+
long fileSize = file.length();
+ // Query performance test
startTime = System.nanoTime();
- try (TsFileSequenceReader sequenceReader = new TsFileSequenceReader(file.getAbsolutePath())) {
- QueryExecutor queryExecutor =
- new TsFileExecutor(
- new MetadataQuerierByFileImpl(sequenceReader),
- new CachedChunkLoaderImpl(sequenceReader));
-
- List<Path> selectedSeries = new ArrayList<>();
- for (int i = 0; i < measurementSchemaCnt; i++) {
- for (int j = 0; j < devicePerTable; j++) {
- selectedSeries.add(new Path(genTreeDeviceId(tableCnt / 2, j), "s" + i, false));
- }
- }
- final QueryExpression queryExpression = QueryExpression.create(selectedSeries, null);
- final QueryDataSet queryDataSet = queryExecutor.execute(queryExpression);
- int cnt = 0;
- while (queryDataSet.hasNext()) {
- queryDataSet.next();
- cnt++;
+ try (ITsFileTreeReader reader = new TsFileTreeReaderBuilder().file(file).build()) {
+ List<String> measurementNames =
+ measurementSchemas.stream()
+ .map(IMeasurementSchema::getMeasurementName)
+ .collect(Collectors.toList());
+
+ ResultSet resultSet =
+ reader.query(allDeviceIds, measurementNames, Long.MIN_VALUE, Long.MAX_VALUE);
+ while (resultSet.next()) {
+ // Consume all results to ensure complete query execution
}
}
queryTimeSum = System.nanoTime() - startTime;
+
+ // Clean up test file
file.delete();
- System.out.printf(
- "Tree register %dns, write %dns, close %dns, query %dns, fileSize %d %n",
- registerTimeSum, writeTimeSum, closeTimeSum, queryTimeSum, fileSize);
+ // Record metrics for this run
registerTimeList.add(registerTimeSum);
writeTimeList.add(writeTimeSum);
closeTimeList.add(closeTimeSum);
@@ -195,18 +302,29 @@ public class PerformanceTest {
fileSizeList.add(fileSize);
}
- private void testTable() throws IOException, WriteProcessException, ReadProcessException {
+ /** Tests the Table mode performance with microsecond timing */
+ private void testTable(
+ List<Long> registerTimeList,
+ List<Long> writeTimeList,
+ List<Long> closeTimeList,
+ List<Long> queryTimeList,
+ List<Long> fileSizeList)
+ throws IOException, WriteProcessException, ReadProcessException {
long registerTimeSum = 0;
long writeTimeSum = 0;
long closeTimeSum = 0;
long queryTimeSum = 0;
long startTime;
- final File file = initFile();
+
+ final File file = initFile("test_table");
TsFileWriter tsFileWriter = new TsFileWriter(file);
try {
+ // Register table schemas
startTime = System.nanoTime();
registerTable(tsFileWriter);
registerTimeSum = System.nanoTime() - startTime;
+
+ // Write data tablets
Tablet tablet = initTableTablet();
for (int tableNum = 0; tableNum < tableCnt; tableNum++) {
for (int deviceNum = 0; deviceNum < devicePerTable; deviceNum++) {
@@ -221,39 +339,36 @@ public class PerformanceTest {
}
}
} finally {
+ // Measure close time
startTime = System.nanoTime();
tsFileWriter.close();
closeTimeSum = System.nanoTime() - startTime;
}
long fileSize = file.length();
+ // Query performance test
startTime = System.nanoTime();
- try (TsFileSequenceReader sequenceReader = new TsFileSequenceReader(file.getAbsolutePath())) {
- TableQueryExecutor tableQueryExecutor =
- new TableQueryExecutor(
- new MetadataQuerierByFileImpl(sequenceReader),
- new CachedChunkLoaderImpl(sequenceReader),
- TableQueryOrdering.DEVICE);
-
- List<String> columns =
- measurementSchemas.stream()
- .map(IMeasurementSchema::getMeasurementName)
- .collect(Collectors.toList());
- TsBlockReader reader =
- tableQueryExecutor.query(genTableName(tableCnt / 2), columns, null, null, null);
- assertTrue(reader.hasNext());
- int cnt = 0;
- while (reader.hasNext()) {
- final TsBlock result = reader.next();
- cnt += result.getPositionCount();
+ for (int tableNum = 0; tableNum < tableCnt; tableNum++) {
+ String tableName = genTableName(tableNum);
+ List<String> columnNames = new ArrayList<>();
+ idSchemas.stream().map(IMeasurementSchema::getMeasurementName).forEach(columnNames::add);
+ measurementSchemas.stream()
+ .map(IMeasurementSchema::getMeasurementName)
+ .forEach(columnNames::add);
+
+ try (ITsFileReader reader = new TsFileReaderBuilder().file(file).build()) {
+ ResultSet resultSet = reader.query(tableName, columnNames, Long.MIN_VALUE, Long.MAX_VALUE);
+ while (resultSet.next()) {
+ // Consume all results to ensure complete query execution
+ }
}
}
- file.delete();
queryTimeSum = System.nanoTime() - startTime;
- System.out.printf(
- "Table register %dns, write %dns, close %dns, query %dns, fileSize %d %n",
- registerTimeSum, writeTimeSum, closeTimeSum, queryTimeSum, fileSize);
+ // Clean up test file
+ file.delete();
+
+ // Record metrics for this run
registerTimeList.add(registerTimeSum);
writeTimeList.add(writeTimeSum);
closeTimeList.add(closeTimeSum);
@@ -261,21 +376,36 @@ public class PerformanceTest {
fileSizeList.add(fileSize);
}
- private File initFile() throws IOException {
- File dir = new File(testDir);
- dir.mkdirs();
- return new File(dir, "testTsFile");
+ // ============ Helper Methods ============
+
+ /** Initialize schemas based on current configuration */
+ private void initSchemas() {
+ idSchemas = new ArrayList<>(idSchemaCnt);
+ for (int i = 0; i < idSchemaCnt; i++) {
+ idSchemas.add(
+ new MeasurementSchema(
+ "id" + i, TSDataType.TEXT, TSEncoding.PLAIN, CompressionType.UNCOMPRESSED));
+ }
+
+ measurementSchemas = new ArrayList<>();
+ for (int i = 0; i < measurementSchemaCnt; i++) {
+ measurementSchemas.add(
+ new MeasurementSchema(
+ "s" + i, TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4));
+ }
}
- private Tablet initTreeTablet() {
- return new Tablet(null, measurementSchemas, pointPerSeries);
+ private File initFile(String prefix) throws IOException {
+ File dir = new File(testDir);
+ dir.mkdirs();
+ return new File(dir, prefix + "_" + System.currentTimeMillis() + ".tsfile");
}
private void fillTreeTablet(Tablet tablet, int tableNum, int deviceNum, int tabletNum) {
tablet.setDeviceId(genTreeDeviceId(tableNum, deviceNum).toString());
for (int i = 0; i < measurementSchemaCnt; i++) {
for (int valNum = 0; valNum < pointPerSeries; valNum++) {
- tablet.addValue(valNum, i, tabletNum * pointPerSeries + valNum);
+ tablet.addValue(valNum, i, (long) tabletNum * pointPerSeries + valNum);
}
}
for (int valNum = 0; valNum < pointPerSeries; valNum++) {
@@ -285,14 +415,20 @@ public class PerformanceTest {
private Tablet initTableTablet() {
List<IMeasurementSchema> allSchema = new ArrayList<>(idSchemas);
- List<ColumnCategory> columnCategories =
- ColumnCategory.nCopy(ColumnCategory.TAG, idSchemas.size());
allSchema.addAll(measurementSchemas);
- columnCategories.addAll(ColumnCategory.nCopy(ColumnCategory.FIELD, measurementSchemaCnt));
+
+ List<ColumnCategory> columnCategories = new ArrayList<>();
+ for (int i = 0; i < idSchemaCnt; i++) {
+ columnCategories.add(ColumnCategory.TAG);
+ }
+ for (int i = 0; i < measurementSchemaCnt; i++) {
+ columnCategories.add(ColumnCategory.FIELD);
+ }
+
return new Tablet(
null,
- IMeasurementSchema.getMeasurementNameList(measurementSchemas),
- IMeasurementSchema.getDataTypeList(measurementSchemas),
+ IMeasurementSchema.getMeasurementNameList(allSchema),
+ IMeasurementSchema.getDataTypeList(allSchema),
columnCategories,
pointPerSeries);
}
@@ -307,7 +443,7 @@ public class PerformanceTest {
}
for (int i = 0; i < measurementSchemaCnt; i++) {
for (int valNum = 0; valNum < pointPerSeries; valNum++) {
- tablet.addValue(valNum, i + idSchemaCnt, tabletNum * pointPerSeries + valNum);
+ tablet.addValue(valNum, i + idSchemaCnt, (long) (tabletNum * pointPerSeries + valNum));
}
}
for (int valNum = 0; valNum < pointPerSeries; valNum++) {
@@ -315,20 +451,26 @@ public class PerformanceTest {
}
}
- private void registerTree(TsFileWriter writer) throws WriteProcessException {
- for (int tableNum = 0; tableNum < tableCnt; tableNum++) {
- for (int deviceNum = 0; deviceNum < devicePerTable; deviceNum++) {
- writer.registerAlignedTimeseries(genTreeDeviceId(tableNum, deviceNum), measurementSchemas);
- }
+ private void registerTable(TsFileWriter writer) {
+ for (int i = 0; i < tableCnt; i++) {
+ TableSchema tableSchema = genTableSchema(i);
+ writer.registerTableSchema(tableSchema);
}
}
- private IMeasurementSchema genMeasurementSchema(int measurementNum) {
- return measurementSchemas.get(measurementNum);
- }
+ private TableSchema genTableSchema(int tableNum) {
+ List<IMeasurementSchema> allSchemas = new ArrayList<>(idSchemas);
+ allSchemas.addAll(measurementSchemas);
+
+ List<ColumnCategory> columnCategories = new ArrayList<>();
+ for (int i = 0; i < idSchemaCnt; i++) {
+ columnCategories.add(ColumnCategory.TAG);
+ }
+ for (int i = 0; i < measurementSchemaCnt; i++) {
+ columnCategories.add(ColumnCategory.FIELD);
+ }
- private IMeasurementSchema genIdSchema(int idNum) {
- return idSchemas.get(idNum);
+ return new TableSchema(genTableName(tableNum), allSchemas, columnCategories);
}
private String genTableName(int tableNum) {
@@ -339,35 +481,19 @@ public class PerformanceTest {
String[] idSegments = new String[idSchemaCnt + 1];
idSegments[0] = genTableName(tableNum);
for (int i = 0; i < idSchemaCnt; i++) {
- idSegments[i + 1] = "0";
+ idSegments[i + 1] = "d0";
}
- idSegments[idSchemaCnt] = Integer.toString(deviceNum);
+ idSegments[idSchemaCnt] = "d" + deviceNum;
return new StringArrayDeviceID(idSegments);
}
private IDeviceID genTreeDeviceId(int tableNum, int deviceNum) {
- return Factory.DEFAULT_FACTORY.create(genTableDeviceId(tableNum, deviceNum).toString());
- }
-
- private void registerTable(TsFileWriter writer) {
- for (int i = 0; i < tableCnt; i++) {
- TableSchema tableSchema = genTableSchema(i);
- writer.registerTableSchema(tableSchema);
- }
- }
-
- private TableSchema genTableSchema(int tableNum) {
- List<IMeasurementSchema> measurementSchemas = new ArrayList<>();
- List<ColumnCategory> columnCategories = new ArrayList<>();
-
+ String[] idSegments = new String[idSchemaCnt + 1];
+ idSegments[0] = genTableName(tableNum);
for (int i = 0; i < idSchemaCnt; i++) {
- measurementSchemas.add(genIdSchema(i));
- columnCategories.add(ColumnCategory.TAG);
- }
- for (int i = 0; i < measurementSchemaCnt; i++) {
- measurementSchemas.add(genMeasurementSchema(i));
- columnCategories.add(ColumnCategory.FIELD);
+ idSegments[i + 1] = "d0";
}
- return new TableSchema(genTableName(tableNum), measurementSchemas, columnCategories);
+ idSegments[idSchemaCnt] = "d" + deviceNum;
+ return new PlainDeviceID(String.join(".", idSegments));
}
}
diff --git a/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java b/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java
index 264bf9e5..6449b52d 100644
--- a/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java
+++ b/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java
@@ -1150,4 +1150,34 @@ public class TsFileWriteApiTest {
Assert.assertTrue(resultSet.isNull(3));
}
}
+
+ @Test
+ public void writeRecord() throws IOException, WriteProcessException, ReadProcessException {
+ setEnv(100 * 1024 * 1024, 10 * 1024);
+
+ TableSchema tableSchema =
+ new TableSchema(
+ "Table1",
+ Arrays.asList(
+ new ColumnSchema("tag1", TSDataType.STRING, ColumnCategory.TAG),
+ new ColumnSchema("field1", TSDataType.BOOLEAN, ColumnCategory.FIELD)));
+ try (ITsFileWriter writer =
+ new TsFileWriterBuilder().file(f).tableSchema(tableSchema).build()) {
+ writer.write(new TSRecord("Table1", 0).addPoint("tag1", "d1").addPoint("field1", true));
+ writer.write(new TSRecord("Table1", 1).addPoint("tag1", "d2").addPoint("field1", false));
+ }
+ try (ITsFileReader reader = new TsFileReaderBuilder().file(f).build();
+ ResultSet resultSet =
+ reader.query(
+ "table1", Arrays.asList("tag1", "field1"), Long.MIN_VALUE, Long.MAX_VALUE)) {
+ Assert.assertTrue(resultSet.next());
+ Assert.assertEquals(0, resultSet.getLong(1));
+ Assert.assertEquals("d1", resultSet.getString(2));
+ Assert.assertTrue(resultSet.getBoolean(3));
+ Assert.assertTrue(resultSet.next());
+ Assert.assertEquals(1, resultSet.getLong(1));
+ Assert.assertEquals("d2", resultSet.getString(2));
+ Assert.assertFalse(resultSet.getBoolean(3));
+ }
+ }
}