You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tsfile.apache.org by ji...@apache.org on 2024/10/18 06:20:01 UTC
(tsfile) branch no_assume_id_column_first created (now 391b5546)
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a change to branch no_assume_id_column_first
in repository https://gitbox.apache.org/repos/asf/tsfile.git
at 391b5546 Remove the assumption that id columns should be the first.
This branch includes the following new commits:
new 391b5546 Remove the assumption that id columns should be the first.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(tsfile) 01/01: Remove the assumption that id columns should be the first.
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch no_assume_id_column_first
in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit 391b55467e8ef5fe40a2f58763394e4e3a953554
Author: Tian Jiang <jt...@163.com>
AuthorDate: Fri Oct 18 14:21:02 2024 +0800
Remove the assumption that id columns should be the first.
---
.../apache/tsfile/file/metadata/TableSchema.java | 37 ++++++++++++++-
.../reader/block/SingleDeviceTsBlockReader.java | 2 +-
.../org/apache/tsfile/tableview/TableViewTest.java | 53 ++++++++++++++++++++--
3 files changed, 85 insertions(+), 7 deletions(-)
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchema.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchema.java
index 474c4a03..805c4834 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchema.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchema.java
@@ -38,6 +38,7 @@ import java.util.Map;
import java.util.Objects;
public class TableSchema {
+
// the tableName is not serialized since the TableSchema is always stored in a Map, from whose
// key the tableName can be known
protected String tableName;
@@ -45,8 +46,10 @@ public class TableSchema {
protected List<ColumnType> columnTypes;
protected boolean updatable = false;
- // columnName -> pos in columnSchemas;
+ // columnName -> pos in columnSchemas
private Map<String, Integer> columnPosIndex;
+ // columnName -> pos in all id columns
+ private Map<String, Integer> idColumnOrder;
public TableSchema(String tableName) {
this.tableName = tableName;
@@ -69,6 +72,16 @@ public class TableSchema {
return columnPosIndex;
}
+ public Map<String, Integer> getIdColumnOrder() {
+ if (idColumnOrder == null) {
+ idColumnOrder = new HashMap<>();
+ }
+ return idColumnOrder;
+ }
+
+ /**
+ * @return i if the given column is the i-th column, -1 if the column is not in the schema
+ */
public int findColumnIndex(String columnName) {
return getColumnPosIndex()
.computeIfAbsent(
@@ -83,6 +96,28 @@ public class TableSchema {
});
}
+ /**
+ * @return i if the given column is the i-th ID column, -1 if the column is not in the schema or
+ * not an ID column
+ */
+ public int findIdColumnOrder(String columnName) {
+ return getIdColumnOrder()
+ .computeIfAbsent(
+ columnName,
+ colName -> {
+ int columnOrder = 0;
+ for (int i = 0; i < columnSchemas.size(); i++) {
+ if (columnSchemas.get(i).getMeasurementId().equals(columnName)
+ && columnTypes.get(i) == ColumnType.ID) {
+ return columnOrder;
+ } else if (columnTypes.get(i) == ColumnType.ID) {
+ columnOrder++;
+ }
+ }
+ return -1;
+ });
+ }
+
public IMeasurementSchema findColumnSchema(String columnName) {
final int columnIndex = findColumnIndex(columnName);
return columnIndex >= 0 ? columnSchemas.get(columnIndex) : null;
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/block/SingleDeviceTsBlockReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/block/SingleDeviceTsBlockReader.java
index 473c3127..2051e958 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/block/SingleDeviceTsBlockReader.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/block/SingleDeviceTsBlockReader.java
@@ -92,7 +92,7 @@ public class SingleDeviceTsBlockReader implements TsBlockReader {
for (String idColumn : task.getColumnMapping().getIdColumns()) {
final List<Integer> columnPosInResult = task.getColumnMapping().getColumnPos(idColumn);
// the first segment in DeviceId is the table name
- final int columnPosInId = task.getTableSchema().findColumnIndex(idColumn) + 1;
+ final int columnPosInId = task.getTableSchema().findIdColumnOrder(idColumn) + 1;
idColumnContextMap.put(idColumn, new IdColumnContext(columnPosInResult, columnPosInId));
}
}
diff --git a/java/tsfile/src/test/java/org/apache/tsfile/tableview/TableViewTest.java b/java/tsfile/src/test/java/org/apache/tsfile/tableview/TableViewTest.java
index 95fcc66b..3332a4af 100644
--- a/java/tsfile/src/test/java/org/apache/tsfile/tableview/TableViewTest.java
+++ b/java/tsfile/src/test/java/org/apache/tsfile/tableview/TableViewTest.java
@@ -93,14 +93,26 @@ public class TableViewTest {
assertEquals(tablet, deserialized);
}
+ @Test
+ public void testWriterWithIDOrderUnfixed()
+ throws IOException, WriteProcessException, ReadProcessException {
+ TableSchema tableSchema = genMixedTableSchema(0);
+ testWrite(tableSchema);
+ }
+
@Test
public void testWriteOneTable() throws IOException, WriteProcessException, ReadProcessException {
+ testWrite(testTableSchema);
+ }
+
+ private void testWrite(TableSchema tableSchema)
+ throws IOException, WriteProcessException, ReadProcessException {
final File testFile = new File(testDir, "testFile");
TsFileWriter writer = new TsFileWriter(testFile);
writer.setGenerateTableSchema(true);
- writer.registerTableSchema(testTableSchema);
+ writer.registerTableSchema(tableSchema);
- writer.writeTable(genTablet(testTableSchema, 0, 100));
+ writer.writeTable(genTablet(tableSchema, 0, 100));
writer.close();
TsFileSequenceReader sequenceReader = new TsFileSequenceReader(testFile.getAbsolutePath());
@@ -111,18 +123,18 @@ public class TableViewTest {
TableQueryOrdering.DEVICE);
final List<String> columns =
- testTableSchema.getColumnSchemas().stream()
+ tableSchema.getColumnSchemas().stream()
.map(IMeasurementSchema::getMeasurementId)
.collect(Collectors.toList());
final TsBlockReader reader =
- tableQueryExecutor.query(testTableSchema.getTableName(), columns, null, null, null);
+ tableQueryExecutor.query(tableSchema.getTableName(), columns, null, null, null);
assertTrue(reader.hasNext());
int cnt = 0;
while (reader.hasNext()) {
final TsBlock result = reader.next();
for (int i = 0; i < result.getPositionCount(); i++) {
String col = result.getColumn(0).getObject(i).toString();
- for (int j = 1; j < testTableSchema.getColumnSchemas().size(); j++) {
+ for (int j = 1; j < tableSchema.getColumnSchemas().size(); j++) {
assertEquals(col, result.getColumn(j).getObject(i).toString());
}
}
@@ -399,4 +411,35 @@ public class TableViewTest {
}
return new TableSchema("testTable" + tableNum, measurementSchemas, columnTypes);
}
+
+ private TableSchema genMixedTableSchema(int tableNum) {
+ List<IMeasurementSchema> measurementSchemas = new ArrayList<>();
+ List<ColumnType> columnTypes = new ArrayList<>();
+
+ int idIndex = 0;
+ int measurementIndex = 0;
+
+ while (idIndex < idSchemaNum || measurementIndex < measurementSchemaNum) {
+ if (idIndex < idSchemaNum) {
+ measurementSchemas.add(
+ new MeasurementSchema(
+ "id" + idIndex, TSDataType.TEXT, TSEncoding.PLAIN, CompressionType.UNCOMPRESSED));
+ columnTypes.add(ColumnType.ID);
+ idIndex++;
+ }
+
+ if (measurementIndex < measurementSchemaNum) {
+ measurementSchemas.add(
+ new MeasurementSchema(
+ "s" + measurementIndex,
+ TSDataType.INT64,
+ TSEncoding.PLAIN,
+ CompressionType.UNCOMPRESSED));
+ columnTypes.add(ColumnType.MEASUREMENT);
+ measurementIndex++;
+ }
+ }
+
+ return new TableSchema("testTable" + tableNum, measurementSchemas, columnTypes);
+ }
}