You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tsfile.apache.org by ja...@apache.org on 2024/08/05 04:11:26 UTC
(tsfile) branch iotdb updated (c8739028 -> af32f498)
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a change to branch iotdb
in repository https://gitbox.apache.org/repos/asf/tsfile.git
from c8739028 upgrade version to a6fb416
new 1210c116 Update Powered-By.md (#45)
new 5acbbd0d Support codecov to show code coverage (#184)
new 9be97f38 Bump org.tukaani:xz from 1.9 to 1.10 (#190)
new 12c37a17 Bump org.apache.commons:commons-lang3 from 3.13.0 to 3.15.0 (#187)
new 684b9bea Bump org.apache.maven.plugins:maven-javadoc-plugin from 3.7.0 to 3.8.0 (#186)
new b45db100 Generate code coverage report for c++ code (#191)
new 5d63035a Bump com.gradle:common-custom-user-data-maven-extension from 1.12.4 to 2 (#188)
new 15ed6590 Fix c++ code coverage didn't show cc files (#193)
new dde6b382 Bump org.codehaus.mojo:exec-maven-plugin from 1.6.0 to 3.3.0 (#189)
new 77db2076 Use pytest to run python test (#194)
new 3f241274 Optimize code coverage structure (#195)
new 8aaedb22 TsFile V4 for Table Model
new af32f498 change version
The 13 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/code-coverage.yml | 40 ++
.github/workflows/unit-test.yml | 4 +-
.mvn/extensions.xml | 2 +-
README-zh.md | 3 +-
README.md | 1 +
docs/src/.vuepress/public/.asf.yaml => codecov.yml | 39 +-
cpp/CMakeLists.txt | 10 +-
cpp/bench_mark/bench_mark_src/bench_conf.h | 4 +-
cpp/examples/c_examples/c_examples.h | 4 +-
cpp/pom.xml | 12 +-
cpp/src/CMakeLists.txt | 14 +-
cpp/src/reader/query_data_set.h | 1 +
cpp/test/CMakeLists.txt | 7 +-
docs/src/zh/Development/Powered-By.md | 2 +-
java/common/pom.xml | 2 +-
.../org/apache/tsfile/block/column/Column.java | 20 +
java/examples/pom.xml | 6 +-
.../tsfile/TsFileWriteAlignedWithTSRecord.java | 4 +-
.../tsfile/TsFileWriteAlignedWithTablet.java | 8 +-
.../org/apache/tsfile/TsFileWriteWithTSRecord.java | 4 +-
.../org/apache/tsfile/TsFileWriteWithTablet.java | 4 +-
java/pom.xml | 9 +-
java/tools/README-zh.md | 126 ++++
java/tools/README.md | 127 ++++
java/tools/pom.xml | 92 +++
.../assembly/resources/conf/logback-cvs2tsfile.xml | 49 ++
.../src/assembly/resources/tools/csv2tsfile.bat | 50 ++
.../src/assembly/resources/tools/csv2tsfile.sh | 53 ++
java/tools/src/assembly/tools.xml | 54 ++
.../org/apache/tsfile/tools/DateTimeUtils.java | 514 +++++++++++++++
.../java/org/apache/tsfile/tools/SchemaParser.java | 264 ++++++++
.../java/org/apache/tsfile/tools/TsFileTool.java | 498 ++++++++++++++
.../org/apache/tsfile/tools/TsfiletoolsTest.java | 158 +++++
java/tsfile/pom.xml | 12 +-
.../org/apache/tsfile/common/cache/LRUCache.java | 4 +
.../apache/tsfile/common/conf/TSFileConfig.java | 7 +-
.../tsfile/compatibility/BufferDeserializer.java | 14 +-
.../tsfile/compatibility/CompatibilityUtils.java | 74 +++
.../tsfile/compatibility/DeserializeConfig.java | 103 +++
.../StreamDeserializer.java} | 10 +-
.../tsfile/exception/IllegalDeviceIDException.java | 15 +-
.../tsfile/exception/TsFileRuntimeException.java | 4 +
.../FileVersionTooOldException.java} | 11 +-
.../tsfile/exception/read/NoColumnException.java | 15 +-
.../ReadProcessException.java} | 16 +-
.../UnsupportedOrderingException.java} | 10 +-
...ception.java => ConflictDataTypeException.java} | 9 +-
...rementException.java => NoDeviceException.java} | 6 +-
.../exception/write/NoMeasurementException.java | 4 +-
.../tsfile/exception/write/NoTableException.java | 15 +-
.../apache/tsfile/file/IMetadataIndexEntry.java | 2 +
.../tsfile/file/header/ChunkGroupHeader.java | 25 +-
.../org/apache/tsfile/file/header/PageHeader.java | 6 +
.../apache/tsfile/file/metadata/ChunkMetadata.java | 26 +
.../file/metadata/DeviceMetadataIndexEntry.java | 18 +-
.../org/apache/tsfile/file/metadata/IDeviceID.java | 134 +++-
.../tsfile/file/metadata/LogicalTableSchema.java | 82 +++
.../metadata/MeasurementMetadataIndexEntry.java | 6 +
.../file/metadata/MetadataIndexConstructor.java | 11 +
.../tsfile/file/metadata/MetadataIndexNode.java | 20 +-
.../apache/tsfile/file/metadata/PlainDeviceID.java | 93 ++-
.../tsfile/file/metadata/StringArrayDeviceID.java | 304 +++++++++
.../apache/tsfile/file/metadata/TableSchema.java | 197 ++++++
.../tsfile/file/metadata/TimeseriesMetadata.java | 16 +
.../tsfile/file/metadata/TsFileMetadata.java | 120 +++-
.../apache/tsfile/read/TsFileSequenceReader.java | 568 +++++++++++-----
...leSequenceReaderTimeseriesMetadataIterator.java | 23 +-
.../java/org/apache/tsfile/read/common/Path.java | 75 +--
.../apache/tsfile/read/common/block/TsBlock.java | 73 ++-
.../tsfile/read/common/block/TsBlockBuilder.java | 4 +
.../tsfile/read/common/block/TsBlockUtil.java | 10 +-
.../read/common/block/column/BinaryColumn.java | 21 +-
.../read/common/block/column/BooleanColumn.java | 21 +-
.../read/common/block/column/ColumnEncoder.java | 5 -
.../read/common/block/column/ColumnFactory.java} | 38 +-
.../read/common/block/column/DoubleColumn.java | 21 +-
.../read/common/block/column/FloatColumn.java | 21 +-
.../block/column/Int64ArrayColumnEncoder.java | 22 -
.../tsfile/read/common/block/column/IntColumn.java | 21 +-
.../read/common/block/column/LongColumn.java | 21 +-
.../read/common/block/column/NullColumn.java | 10 +-
.../block/column/RunLengthEncodedColumn.java | 12 +-
.../read/common/block/column/TimeColumn.java | 20 +-
.../read/common/block/column/TsBlockSerde.java | 6 +-
.../read/common/parser/PathNodesGenerator.java | 3 +
.../type/{IntType.java => AbstractIntType.java} | 21 +-
.../type/{LongType.java => AbstractLongType.java} | 21 +-
.../tsfile/read/common/type/AbstractType.java | 16 +-
.../{BinaryType.java => AbstractVarcharType.java} | 21 +-
.../apache/tsfile/read/common/type/BinaryType.java | 29 +-
.../common/type/{BinaryType.java => BlobType.java} | 36 +-
.../tsfile/read/common/type/BooleanType.java | 29 +-
.../common/type/{TypeEnum.java => DateType.java} | 21 +-
.../apache/tsfile/read/common/type/DoubleType.java | 29 +-
.../apache/tsfile/read/common/type/FloatType.java | 29 +-
.../apache/tsfile/read/common/type/IntType.java | 58 +-
.../apache/tsfile/read/common/type/LongType.java | 58 +-
.../apache/tsfile/read/common/type/RowType.java | 168 +++++
.../common/type/{TypeEnum.java => StringType.java} | 21 +-
.../type/{TypeEnum.java => TimestampType.java} | 21 +-
.../org/apache/tsfile/read/common/type/Type.java | 14 +
.../apache/tsfile/read/common/type/TypeEnum.java | 10 +-
.../tsfile/read/common/type/TypeFactory.java | 40 +-
.../tsfile/read/common/type/UnknownType.java | 87 +++
.../tsfile/read/controller/DeviceMetaIterator.java | 133 ++++
.../tsfile/read/controller/IMetadataQuerier.java | 20 +
.../read/controller/MetadataQuerierByFileImpl.java | 65 +-
.../{IUnaryExpression.java => ExpressionTree.java} | 9 +-
.../tsfile/read/filter/factory/FilterFactory.java | 18 +
.../read/query/executor/TableQueryExecutor.java | 163 +++++
.../read/query/executor/task/DeviceQueryTask.java | 73 +++
.../query/executor/task/DeviceTaskIterator.java | 62 ++
.../reader/block/DeviceOrderedTsBlockReader.java | 96 +++
.../reader/block/SingleDeviceTsBlockReader.java | 408 ++++++++++++
.../TsBlockReader.java} | 31 +-
.../reader/series/AbstractFileSeriesReader.java | 6 +
.../read/reader/series/FileSeriesReader.java | 3 +
.../org/apache/tsfile/utils/MeasurementGroup.java | 14 +-
.../tsfile/utils/ReadWriteForEncodingUtils.java | 12 +
.../org/apache/tsfile/utils/ReadWriteIOUtils.java | 6 +-
.../apache/tsfile/utils/TsFileGeneratorUtils.java | 16 +-
.../org/apache/tsfile/utils/TsFileSketchTool.java | 730 +++++++++++++++++++++
.../java/org/apache/tsfile/utils/WriteUtils.java | 64 ++
.../java/org/apache/tsfile/write/TsFileWriter.java | 358 +++++-----
.../write/chunk/AlignedChunkGroupWriterImpl.java | 52 +-
.../tsfile/write/chunk/IChunkGroupWriter.java | 16 +-
.../chunk/NonAlignedChunkGroupWriterImpl.java | 28 +-
.../org/apache/tsfile/write/record/TSRecord.java | 8 +-
.../org/apache/tsfile/write/record/Tablet.java | 268 ++++++--
.../org/apache/tsfile/write/schema/Schema.java | 80 ++-
.../write/writer/RestorableTsFileIOWriter.java | 13 +-
.../apache/tsfile/write/writer/TsFileIOWriter.java | 72 +-
.../write/writer/TsFileIOWriterEndFileTest.java | 4 +-
.../write/writer/tsmiterator/DiskTSMIterator.java | 6 +-
.../write/writer/tsmiterator/TSMIterator.java | 11 +-
.../tsfile/common/block/TsBlockSerdeTest.java | 2 -
.../tsfile/compatibility/CompatibilityTest.java | 74 +++
.../tsfile/exception/TsFileExceptionTest.java | 2 +-
.../apache/tsfile/file/metadata/IDeviceIDTest.java | 143 ++++
.../tsfile/file/metadata/TsFileMetadataTest.java | 4 +-
.../tsfile/file/metadata/utils/TestHelper.java | 14 +-
.../apache/tsfile/file/metadata/utils/Utils.java | 10 +-
.../org/apache/tsfile/read/GetAllDevicesTest.java | 14 +-
...easurementChunkMetadataListMapIteratorTest.java | 7 +-
.../tsfile/read/TimePlainEncodeReadTest.java | 3 +
.../org/apache/tsfile/read/TsFileReaderTest.java | 9 +-
.../tsfile/read/TsFileSequenceReaderTest.java | 61 +-
.../org/apache/tsfile/read/common/PathTest.java | 64 +-
.../AlignedChunkReaderWithoutStatisticsTest.java | 4 +-
.../apache/tsfile/tableview/PerformanceTest.java | 372 +++++++++++
.../apache/tsfile/tableview/TableSchemaTest.java | 123 ++++
.../org/apache/tsfile/tableview/TableViewTest.java | 402 ++++++++++++
.../org/apache/tsfile/utils/FileGenerator.java | 6 +-
.../utils/ReadWriteForEncodingUtilsTest.java | 8 +-
.../java/org/apache/tsfile/utils/RecordUtils.java | 5 +-
.../org/apache/tsfile/utils/RecordUtilsTest.java | 9 +-
.../tsfile/utils/TsFileGeneratorForTest.java | 5 +-
.../tsfile/write/DefaultSchemaTemplateTest.java | 5 +-
.../tsfile/write/MetadataIndexConstructorTest.java | 31 +-
.../apache/tsfile/write/TsFileIOWriterTest.java | 24 +-
.../apache/tsfile/write/TsFileReadWriteTest.java | 18 +-
.../apache/tsfile/write/TsFileWriteApiTest.java | 36 +-
.../org/apache/tsfile/write/TsFileWriterTest.java | 11 +-
.../java/org/apache/tsfile/write/WriteTest.java | 11 +-
.../org/apache/tsfile/write/record/TabletTest.java | 5 +-
.../write/schema/converter/SchemaBuilderTest.java | 4 +-
.../write/writer/RestorableTsFileIOWriterTest.java | 8 +-
.../writer/TsFileIOWriterMemoryControlTest.java | 13 +-
java/tsfile/src/test/resources/v3TsFile | Bin 0 -> 378 bytes
pom.xml | 190 +++++-
python/pom.xml | 22 +-
python/test.py | 2 +-
172 files changed, 8701 insertions(+), 1163 deletions(-)
create mode 100644 .github/workflows/code-coverage.yml
copy docs/src/.vuepress/public/.asf.yaml => codecov.yml (61%)
create mode 100644 java/tools/README-zh.md
create mode 100644 java/tools/README.md
create mode 100644 java/tools/pom.xml
create mode 100644 java/tools/src/assembly/resources/conf/logback-cvs2tsfile.xml
create mode 100644 java/tools/src/assembly/resources/tools/csv2tsfile.bat
create mode 100644 java/tools/src/assembly/resources/tools/csv2tsfile.sh
create mode 100644 java/tools/src/assembly/tools.xml
create mode 100644 java/tools/src/main/java/org/apache/tsfile/tools/DateTimeUtils.java
create mode 100644 java/tools/src/main/java/org/apache/tsfile/tools/SchemaParser.java
create mode 100644 java/tools/src/main/java/org/apache/tsfile/tools/TsFileTool.java
create mode 100644 java/tools/src/test/java/org/apache/tsfile/tools/TsfiletoolsTest.java
copy cpp/examples/c_examples/c_examples.h => java/tsfile/src/main/java/org/apache/tsfile/compatibility/BufferDeserializer.java (82%)
create mode 100644 java/tsfile/src/main/java/org/apache/tsfile/compatibility/CompatibilityUtils.java
create mode 100644 java/tsfile/src/main/java/org/apache/tsfile/compatibility/DeserializeConfig.java
copy java/tsfile/src/main/java/org/apache/tsfile/{exception/write/TsFileNotCompleteException.java => compatibility/StreamDeserializer.java} (80%)
copy cpp/examples/c_examples/c_examples.h => java/tsfile/src/main/java/org/apache/tsfile/exception/IllegalDeviceIDException.java (82%)
copy java/tsfile/src/main/java/org/apache/tsfile/exception/{write/TsFileNotCompleteException.java => read/FileVersionTooOldException.java} (71%)
copy cpp/examples/c_examples/c_examples.h => java/tsfile/src/main/java/org/apache/tsfile/exception/read/NoColumnException.java (79%)
copy java/tsfile/src/main/java/org/apache/tsfile/exception/{NotImplementedException.java => read/ReadProcessException.java} (73%)
copy java/tsfile/src/main/java/org/apache/tsfile/exception/{filter/UnSupportFilterDataTypeException.java => read/UnsupportedOrderingException.java} (78%)
copy java/tsfile/src/main/java/org/apache/tsfile/exception/write/{TsFileNotCompleteException.java => ConflictDataTypeException.java} (73%)
copy java/tsfile/src/main/java/org/apache/tsfile/exception/write/{NoMeasurementException.java => NoDeviceException.java} (85%)
copy cpp/examples/c_examples/c_examples.h => java/tsfile/src/main/java/org/apache/tsfile/exception/write/NoTableException.java (79%)
create mode 100644 java/tsfile/src/main/java/org/apache/tsfile/file/metadata/LogicalTableSchema.java
create mode 100644 java/tsfile/src/main/java/org/apache/tsfile/file/metadata/StringArrayDeviceID.java
create mode 100644 java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchema.java
copy java/{examples/src/main/java/org/apache/tsfile/DataGenerator.java => tsfile/src/main/java/org/apache/tsfile/read/common/block/column/ColumnFactory.java} (61%)
copy java/tsfile/src/main/java/org/apache/tsfile/read/common/type/{IntType.java => AbstractIntType.java} (86%)
copy java/tsfile/src/main/java/org/apache/tsfile/read/common/type/{LongType.java => AbstractLongType.java} (86%)
copy cpp/examples/c_examples/c_examples.h => java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java (82%)
copy java/tsfile/src/main/java/org/apache/tsfile/read/common/type/{BinaryType.java => AbstractVarcharType.java} (81%)
copy java/tsfile/src/main/java/org/apache/tsfile/read/common/type/{BinaryType.java => BlobType.java} (72%)
copy java/tsfile/src/main/java/org/apache/tsfile/read/common/type/{TypeEnum.java => DateType.java} (72%)
create mode 100644 java/tsfile/src/main/java/org/apache/tsfile/read/common/type/RowType.java
copy java/tsfile/src/main/java/org/apache/tsfile/read/common/type/{TypeEnum.java => StringType.java} (70%)
copy java/tsfile/src/main/java/org/apache/tsfile/read/common/type/{TypeEnum.java => TimestampType.java} (69%)
create mode 100644 java/tsfile/src/main/java/org/apache/tsfile/read/common/type/UnknownType.java
create mode 100644 java/tsfile/src/main/java/org/apache/tsfile/read/controller/DeviceMetaIterator.java
copy java/tsfile/src/main/java/org/apache/tsfile/read/expression/{IUnaryExpression.java => ExpressionTree.java} (82%)
create mode 100644 java/tsfile/src/main/java/org/apache/tsfile/read/query/executor/TableQueryExecutor.java
create mode 100644 java/tsfile/src/main/java/org/apache/tsfile/read/query/executor/task/DeviceQueryTask.java
create mode 100644 java/tsfile/src/main/java/org/apache/tsfile/read/query/executor/task/DeviceTaskIterator.java
create mode 100644 java/tsfile/src/main/java/org/apache/tsfile/read/reader/block/DeviceOrderedTsBlockReader.java
create mode 100644 java/tsfile/src/main/java/org/apache/tsfile/read/reader/block/SingleDeviceTsBlockReader.java
copy java/tsfile/src/main/java/org/apache/tsfile/read/reader/{chunk/ChunkReaderWithoutStatistics.java => block/TsBlockReader.java} (62%)
create mode 100644 java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileSketchTool.java
create mode 100644 java/tsfile/src/main/java/org/apache/tsfile/utils/WriteUtils.java
create mode 100644 java/tsfile/src/test/java/org/apache/tsfile/compatibility/CompatibilityTest.java
create mode 100644 java/tsfile/src/test/java/org/apache/tsfile/file/metadata/IDeviceIDTest.java
create mode 100644 java/tsfile/src/test/java/org/apache/tsfile/tableview/PerformanceTest.java
create mode 100644 java/tsfile/src/test/java/org/apache/tsfile/tableview/TableSchemaTest.java
create mode 100644 java/tsfile/src/test/java/org/apache/tsfile/tableview/TableViewTest.java
create mode 100644 java/tsfile/src/test/resources/v3TsFile
(tsfile) 12/13: TsFile V4 for Table Model
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch iotdb
in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit 8aaedb2251d86a5cfb5e54e0db162291b5ba7ffb
Author: Jiang Tian <jt...@163.com>
AuthorDate: Mon Aug 5 10:27:44 2024 +0800
TsFile V4 for Table Model
---
cpp/bench_mark/bench_mark_src/bench_conf.h | 4 +-
cpp/examples/c_examples/c_examples.h | 4 +-
cpp/pom.xml | 2 +-
cpp/src/reader/query_data_set.h | 1 +
java/common/pom.xml | 2 +-
.../org/apache/tsfile/block/column/Column.java | 20 +
java/examples/pom.xml | 4 +-
.../tsfile/TsFileWriteAlignedWithTSRecord.java | 4 +-
.../tsfile/TsFileWriteAlignedWithTablet.java | 8 +-
.../org/apache/tsfile/TsFileWriteWithTSRecord.java | 4 +-
.../org/apache/tsfile/TsFileWriteWithTablet.java | 4 +-
java/pom.xml | 5 +-
java/tools/README-zh.md | 126 ++++
java/tools/README.md | 127 ++++
java/tools/pom.xml | 92 +++
.../assembly/resources/conf/logback-cvs2tsfile.xml | 49 ++
.../src/assembly/resources/tools/csv2tsfile.bat | 50 ++
.../src/assembly/resources/tools/csv2tsfile.sh | 53 ++
java/tools/src/assembly/tools.xml | 54 ++
.../org/apache/tsfile/tools/DateTimeUtils.java | 514 +++++++++++++++
.../java/org/apache/tsfile/tools/SchemaParser.java | 264 ++++++++
.../java/org/apache/tsfile/tools/TsFileTool.java | 498 ++++++++++++++
.../org/apache/tsfile/tools/TsfiletoolsTest.java | 158 +++++
java/tsfile/pom.xml | 12 +-
.../org/apache/tsfile/common/cache/LRUCache.java | 4 +
.../apache/tsfile/common/conf/TSFileConfig.java | 7 +-
.../tsfile/compatibility/BufferDeserializer.java | 14 +-
.../tsfile/compatibility/CompatibilityUtils.java | 74 +++
.../tsfile/compatibility/DeserializeConfig.java | 103 +++
.../StreamDeserializer.java} | 17 +-
.../tsfile/exception/IllegalDeviceIDException.java | 15 +-
.../tsfile/exception/TsFileRuntimeException.java | 4 +
.../read/FileVersionTooOldException.java} | 20 +-
.../tsfile/exception/read/NoColumnException.java | 15 +-
.../ReadProcessException.java} | 17 +-
.../read/UnsupportedOrderingException.java} | 17 +-
...ception.java => ConflictDataTypeException.java} | 10 +-
...rementException.java => NoDeviceException.java} | 6 +-
.../exception/write/NoMeasurementException.java | 4 +-
.../tsfile/exception/write/NoTableException.java | 15 +-
.../apache/tsfile/file/IMetadataIndexEntry.java | 2 +
.../tsfile/file/header/ChunkGroupHeader.java | 25 +-
.../org/apache/tsfile/file/header/PageHeader.java | 6 +
.../apache/tsfile/file/metadata/ChunkMetadata.java | 26 +
.../file/metadata/DeviceMetadataIndexEntry.java | 18 +-
.../org/apache/tsfile/file/metadata/IDeviceID.java | 134 +++-
.../tsfile/file/metadata/LogicalTableSchema.java | 82 +++
.../metadata/MeasurementMetadataIndexEntry.java | 6 +
.../file/metadata/MetadataIndexConstructor.java | 11 +
.../tsfile/file/metadata/MetadataIndexNode.java | 20 +-
.../apache/tsfile/file/metadata/PlainDeviceID.java | 93 ++-
.../tsfile/file/metadata/StringArrayDeviceID.java | 304 +++++++++
.../apache/tsfile/file/metadata/TableSchema.java | 197 ++++++
.../tsfile/file/metadata/TimeseriesMetadata.java | 16 +
.../tsfile/file/metadata/TsFileMetadata.java | 120 +++-
.../apache/tsfile/read/TsFileSequenceReader.java | 568 +++++++++++-----
...leSequenceReaderTimeseriesMetadataIterator.java | 23 +-
.../java/org/apache/tsfile/read/common/Path.java | 75 +--
.../apache/tsfile/read/common/block/TsBlock.java | 73 ++-
.../tsfile/read/common/block/TsBlockBuilder.java | 4 +
.../tsfile/read/common/block/TsBlockUtil.java | 10 +-
.../read/common/block/column/BinaryColumn.java | 21 +-
.../read/common/block/column/BooleanColumn.java | 21 +-
.../read/common/block/column/ColumnEncoder.java | 5 -
.../column/ColumnFactory.java} | 39 +-
.../read/common/block/column/DoubleColumn.java | 21 +-
.../read/common/block/column/FloatColumn.java | 21 +-
.../block/column/Int64ArrayColumnEncoder.java | 22 -
.../tsfile/read/common/block/column/IntColumn.java | 21 +-
.../read/common/block/column/LongColumn.java | 21 +-
.../read/common/block/column/NullColumn.java | 10 +-
.../block/column/RunLengthEncodedColumn.java | 12 +-
.../read/common/block/column/TimeColumn.java | 20 +-
.../read/common/block/column/TsBlockSerde.java | 6 +-
.../read/common/parser/PathNodesGenerator.java | 3 +
.../type/{IntType.java => AbstractIntType.java} | 21 +-
.../type/{LongType.java => AbstractLongType.java} | 21 +-
.../tsfile/read/common/type/AbstractType.java | 16 +-
.../{BinaryType.java => AbstractVarcharType.java} | 21 +-
.../apache/tsfile/read/common/type/BinaryType.java | 29 +-
.../common/type/{BinaryType.java => BlobType.java} | 36 +-
.../tsfile/read/common/type/BooleanType.java | 29 +-
.../common/type/{TypeEnum.java => DateType.java} | 21 +-
.../apache/tsfile/read/common/type/DoubleType.java | 29 +-
.../apache/tsfile/read/common/type/FloatType.java | 29 +-
.../apache/tsfile/read/common/type/IntType.java | 58 +-
.../apache/tsfile/read/common/type/LongType.java | 58 +-
.../apache/tsfile/read/common/type/RowType.java | 168 +++++
.../common/type/{TypeEnum.java => StringType.java} | 21 +-
.../type/{TypeEnum.java => TimestampType.java} | 21 +-
.../org/apache/tsfile/read/common/type/Type.java | 14 +
.../apache/tsfile/read/common/type/TypeEnum.java | 10 +-
.../tsfile/read/common/type/TypeFactory.java | 40 +-
.../tsfile/read/common/type/UnknownType.java | 87 +++
.../tsfile/read/controller/DeviceMetaIterator.java | 133 ++++
.../tsfile/read/controller/IMetadataQuerier.java | 20 +
.../read/controller/MetadataQuerierByFileImpl.java | 65 +-
.../tsfile/read/expression/ExpressionTree.java | 14 +-
.../tsfile/read/filter/factory/FilterFactory.java | 18 +
.../read/query/executor/TableQueryExecutor.java | 163 +++++
.../read/query/executor/task/DeviceQueryTask.java | 73 +++
.../query/executor/task/DeviceTaskIterator.java | 62 ++
.../reader/block/DeviceOrderedTsBlockReader.java | 96 +++
.../reader/block/SingleDeviceTsBlockReader.java | 408 ++++++++++++
.../reader/block/TsBlockReader.java} | 28 +-
.../reader/series/AbstractFileSeriesReader.java | 6 +
.../read/reader/series/FileSeriesReader.java | 3 +
.../org/apache/tsfile/utils/MeasurementGroup.java | 14 +-
.../tsfile/utils/ReadWriteForEncodingUtils.java | 12 +
.../org/apache/tsfile/utils/ReadWriteIOUtils.java | 6 +-
.../apache/tsfile/utils/TsFileGeneratorUtils.java | 16 +-
.../org/apache/tsfile/utils/TsFileSketchTool.java | 730 +++++++++++++++++++++
.../java/org/apache/tsfile/utils/WriteUtils.java | 64 ++
.../java/org/apache/tsfile/write/TsFileWriter.java | 358 +++++-----
.../write/chunk/AlignedChunkGroupWriterImpl.java | 52 +-
.../tsfile/write/chunk/IChunkGroupWriter.java | 16 +-
.../chunk/NonAlignedChunkGroupWriterImpl.java | 28 +-
.../org/apache/tsfile/write/record/TSRecord.java | 8 +-
.../org/apache/tsfile/write/record/Tablet.java | 268 ++++++--
.../org/apache/tsfile/write/schema/Schema.java | 80 ++-
.../write/writer/RestorableTsFileIOWriter.java | 13 +-
.../apache/tsfile/write/writer/TsFileIOWriter.java | 72 +-
.../write/writer/TsFileIOWriterEndFileTest.java | 4 +-
.../write/writer/tsmiterator/DiskTSMIterator.java | 6 +-
.../write/writer/tsmiterator/TSMIterator.java | 11 +-
.../tsfile/common/block/TsBlockSerdeTest.java | 2 -
.../tsfile/compatibility/CompatibilityTest.java | 74 +++
.../tsfile/exception/TsFileExceptionTest.java | 2 +-
.../apache/tsfile/file/metadata/IDeviceIDTest.java | 143 ++++
.../tsfile/file/metadata/TsFileMetadataTest.java | 4 +-
.../tsfile/file/metadata/utils/TestHelper.java | 14 +-
.../apache/tsfile/file/metadata/utils/Utils.java | 10 +-
.../org/apache/tsfile/read/GetAllDevicesTest.java | 14 +-
...easurementChunkMetadataListMapIteratorTest.java | 7 +-
.../tsfile/read/TimePlainEncodeReadTest.java | 3 +
.../org/apache/tsfile/read/TsFileReaderTest.java | 9 +-
.../tsfile/read/TsFileSequenceReaderTest.java | 61 +-
.../org/apache/tsfile/read/common/PathTest.java | 64 +-
.../AlignedChunkReaderWithoutStatisticsTest.java | 4 +-
.../apache/tsfile/tableview/PerformanceTest.java | 372 +++++++++++
.../apache/tsfile/tableview/TableSchemaTest.java | 123 ++++
.../org/apache/tsfile/tableview/TableViewTest.java | 402 ++++++++++++
.../org/apache/tsfile/utils/FileGenerator.java | 6 +-
.../utils/ReadWriteForEncodingUtilsTest.java | 8 +-
.../java/org/apache/tsfile/utils/RecordUtils.java | 5 +-
.../org/apache/tsfile/utils/RecordUtilsTest.java | 9 +-
.../tsfile/utils/TsFileGeneratorForTest.java | 5 +-
.../tsfile/write/DefaultSchemaTemplateTest.java | 5 +-
.../tsfile/write/MetadataIndexConstructorTest.java | 31 +-
.../apache/tsfile/write/TsFileIOWriterTest.java | 24 +-
.../apache/tsfile/write/TsFileReadWriteTest.java | 18 +-
.../apache/tsfile/write/TsFileWriteApiTest.java | 36 +-
.../org/apache/tsfile/write/TsFileWriterTest.java | 11 +-
.../java/org/apache/tsfile/write/WriteTest.java | 11 +-
.../org/apache/tsfile/write/record/TabletTest.java | 5 +-
.../write/schema/converter/SchemaBuilderTest.java | 4 +-
.../write/writer/RestorableTsFileIOWriterTest.java | 8 +-
.../writer/TsFileIOWriterMemoryControlTest.java | 13 +-
java/tsfile/src/test/resources/v3TsFile | Bin 0 -> 378 bytes
pom.xml | 2 +-
python/pom.xml | 2 +-
161 files changed, 8394 insertions(+), 1150 deletions(-)
diff --git a/cpp/bench_mark/bench_mark_src/bench_conf.h b/cpp/bench_mark/bench_mark_src/bench_conf.h
index 9d961d72..f2a4de75 100644
--- a/cpp/bench_mark/bench_mark_src/bench_conf.h
+++ b/cpp/bench_mark/bench_mark_src/bench_conf.h
@@ -16,6 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
+
#include <vector>
namespace bench {
@@ -23,4 +24,5 @@ int LOOP_NUM = 100000;
int THREAD_NUM = 1;
int TIMESERIES_NUM = 50;
std::vector<int> TYPE_LIST = {0, 0, 1, 0, 1};
-} // namespace bench
\ No newline at end of file
+} // namespace bench
+
diff --git a/cpp/examples/c_examples/c_examples.h b/cpp/examples/c_examples/c_examples.h
index d29090cb..a0405138 100644
--- a/cpp/examples/c_examples/c_examples.h
+++ b/cpp/examples/c_examples/c_examples.h
@@ -17,7 +17,6 @@
* under the License.
*/
-
#include "cwrapper/TsFile-cwrapper.h"
#ifdef __cplusplus
@@ -29,4 +28,5 @@ ErrorCode read_tsfile();
#ifdef __cplusplus
}
-#endif
\ No newline at end of file
+#endif
+
diff --git a/cpp/pom.xml b/cpp/pom.xml
index 2527014d..65324616 100644
--- a/cpp/pom.xml
+++ b/cpp/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.tsfile</groupId>
<artifactId>tsfile-parent</artifactId>
- <version>1.0.1-a6fb416-SNAPSHOT</version>
+ <version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>tsfile-cpp</artifactId>
<packaging>pom</packaging>
diff --git a/cpp/src/reader/query_data_set.h b/cpp/src/reader/query_data_set.h
index 85842766..abea04af 100644
--- a/cpp/src/reader/query_data_set.h
+++ b/cpp/src/reader/query_data_set.h
@@ -34,3 +34,4 @@ class QueryDataSet {
} // namespace storage
#endif // READER_QUERY_DATA_SET_H
+
diff --git a/java/common/pom.xml b/java/common/pom.xml
index 3a030ced..fe503903 100644
--- a/java/common/pom.xml
+++ b/java/common/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.tsfile</groupId>
<artifactId>tsfile-java</artifactId>
- <version>1.0.1-a6fb416-SNAPSHOT</version>
+ <version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>common</artifactId>
<name>TsFile: Java: Common</name>
diff --git a/java/common/src/main/java/org/apache/tsfile/block/column/Column.java b/java/common/src/main/java/org/apache/tsfile/block/column/Column.java
index 161a2c8e..f92d3b5c 100644
--- a/java/common/src/main/java/org/apache/tsfile/block/column/Column.java
+++ b/java/common/src/main/java/org/apache/tsfile/block/column/Column.java
@@ -23,6 +23,8 @@ import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.utils.Binary;
import org.apache.tsfile.utils.TsPrimitiveType;
+import java.util.Arrays;
+
public interface Column {
/** Get the data type. */
@@ -124,6 +126,14 @@ public interface Column {
/** Returns the array to determine whether each position of the column is null or not. */
boolean[] isNull();
+ /**
+ * Set the given range as null.
+ *
+ * @param start start position (inclusive)
+ * @param end end position (exclusive)
+ */
+ void setNull(int start, int end);
+
/** Returns the number of positions in this block. */
int getPositionCount();
@@ -164,4 +174,14 @@ public interface Column {
void reverse();
int getInstanceSize();
+
+ void setPositionCount(int count);
+
+ default void reset() {
+ setPositionCount(0);
+ final boolean[] isNulls = isNull();
+ if (isNulls != null) {
+ Arrays.fill(isNulls, false);
+ }
+ }
}
diff --git a/java/examples/pom.xml b/java/examples/pom.xml
index 837be06a..b27fd5bf 100644
--- a/java/examples/pom.xml
+++ b/java/examples/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.tsfile</groupId>
<artifactId>tsfile-java</artifactId>
- <version>1.0.1-a6fb416-SNAPSHOT</version>
+ <version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>examples</artifactId>
<packaging>pom</packaging>
@@ -38,7 +38,7 @@
<dependency>
<groupId>org.apache.tsfile</groupId>
<artifactId>tsfile</artifactId>
- <version>1.0.1-a6fb416-SNAPSHOT</version>
+ <version>1.2.0-SNAPSHOT</version>
</dependency>
</dependencies>
<build>
diff --git a/java/examples/src/main/java/org/apache/tsfile/TsFileWriteAlignedWithTSRecord.java b/java/examples/src/main/java/org/apache/tsfile/TsFileWriteAlignedWithTSRecord.java
index d5a5c523..f7d1bc7b 100644
--- a/java/examples/src/main/java/org/apache/tsfile/TsFileWriteAlignedWithTSRecord.java
+++ b/java/examples/src/main/java/org/apache/tsfile/TsFileWriteAlignedWithTSRecord.java
@@ -55,7 +55,7 @@ public class TsFileWriteAlignedWithTSRecord {
}
try (TsFileWriter tsFileWriter = new TsFileWriter(f)) {
- List<MeasurementSchema> measurementSchemas = new ArrayList<>();
+ List<IMeasurementSchema> measurementSchemas = new ArrayList<>();
measurementSchemas.add(
new MeasurementSchema(Constant.SENSOR_1, TSDataType.INT64, TSEncoding.RLE));
measurementSchemas.add(
@@ -84,7 +84,7 @@ public class TsFileWriteAlignedWithTSRecord {
private static void writeAligned(
TsFileWriter tsFileWriter,
String deviceId,
- List<MeasurementSchema> schemas,
+ List<IMeasurementSchema> schemas,
long rowSize,
long startTime,
long startValue)
diff --git a/java/examples/src/main/java/org/apache/tsfile/TsFileWriteAlignedWithTablet.java b/java/examples/src/main/java/org/apache/tsfile/TsFileWriteAlignedWithTablet.java
index 7f7f6f36..73704ca0 100644
--- a/java/examples/src/main/java/org/apache/tsfile/TsFileWriteAlignedWithTablet.java
+++ b/java/examples/src/main/java/org/apache/tsfile/TsFileWriteAlignedWithTablet.java
@@ -26,6 +26,7 @@ import org.apache.tsfile.fileSystem.FSFactoryProducer;
import org.apache.tsfile.read.common.Path;
import org.apache.tsfile.write.TsFileWriter;
import org.apache.tsfile.write.record.Tablet;
+import org.apache.tsfile.write.schema.IMeasurementSchema;
import org.apache.tsfile.write.schema.MeasurementSchema;
import org.slf4j.Logger;
@@ -56,7 +57,8 @@ public class TsFileWriteAlignedWithTablet {
}
try (TsFileWriter tsFileWriter = new TsFileWriter(f)) {
- List<MeasurementSchema> measurementSchemas = new ArrayList<>();
+
+ List<IMeasurementSchema> measurementSchemas = new ArrayList<>();
measurementSchemas.add(
new MeasurementSchema(Constant.SENSOR_1, TSDataType.INT64, TSEncoding.PLAIN));
measurementSchemas.add(
@@ -87,7 +89,7 @@ public class TsFileWriteAlignedWithTablet {
private static void writeAlignedWithTablet(
TsFileWriter tsFileWriter,
String deviceId,
- List<MeasurementSchema> schemas,
+ List<IMeasurementSchema> schemas,
long rowNum,
long startTime,
long startValue)
@@ -127,7 +129,7 @@ public class TsFileWriteAlignedWithTablet {
tsFileWriter.registerTimeseries(
new Path(DEVICE_2), new MeasurementSchema(SENSOR_2, TSDataType.INT64, TSEncoding.RLE));
// construct Tablet
- List<MeasurementSchema> measurementSchemas = new ArrayList<>();
+ List<IMeasurementSchema> measurementSchemas = new ArrayList<>();
measurementSchemas.add(new MeasurementSchema(SENSOR_1, TSDataType.INT64, TSEncoding.RLE));
measurementSchemas.add(new MeasurementSchema(SENSOR_2, TSDataType.INT64, TSEncoding.RLE));
Tablet tablet = new Tablet(DEVICE_2, measurementSchemas);
diff --git a/java/examples/src/main/java/org/apache/tsfile/TsFileWriteWithTSRecord.java b/java/examples/src/main/java/org/apache/tsfile/TsFileWriteWithTSRecord.java
index 4234c830..79f9fd52 100644
--- a/java/examples/src/main/java/org/apache/tsfile/TsFileWriteWithTSRecord.java
+++ b/java/examples/src/main/java/org/apache/tsfile/TsFileWriteWithTSRecord.java
@@ -56,7 +56,7 @@ public class TsFileWriteWithTSRecord {
}
try (TsFileWriter tsFileWriter = new TsFileWriter(f)) {
- List<MeasurementSchema> schemas = new ArrayList<>();
+ List<IMeasurementSchema> schemas = new ArrayList<>();
schemas.add(new MeasurementSchema(Constant.SENSOR_1, TSDataType.INT64, TSEncoding.RLE));
schemas.add(new MeasurementSchema(Constant.SENSOR_2, TSDataType.INT64, TSEncoding.RLE));
schemas.add(new MeasurementSchema(Constant.SENSOR_3, TSDataType.INT64, TSEncoding.RLE));
@@ -80,7 +80,7 @@ public class TsFileWriteWithTSRecord {
private static void write(
TsFileWriter tsFileWriter,
String deviceId,
- List<MeasurementSchema> schemas,
+ List<IMeasurementSchema> schemas,
long rowSize,
long startTime,
long startValue)
diff --git a/java/examples/src/main/java/org/apache/tsfile/TsFileWriteWithTablet.java b/java/examples/src/main/java/org/apache/tsfile/TsFileWriteWithTablet.java
index a9c572e2..3ab95a18 100644
--- a/java/examples/src/main/java/org/apache/tsfile/TsFileWriteWithTablet.java
+++ b/java/examples/src/main/java/org/apache/tsfile/TsFileWriteWithTablet.java
@@ -51,7 +51,7 @@ public class TsFileWriteWithTablet {
}
try (TsFileWriter tsFileWriter = new TsFileWriter(f)) {
- List<MeasurementSchema> measurementSchemas = new ArrayList<>();
+ List<IMeasurementSchema> measurementSchemas = new ArrayList<>();
measurementSchemas.add(
new MeasurementSchema(Constant.SENSOR_1, TSDataType.INT64, TSEncoding.PLAIN));
measurementSchemas.add(
@@ -81,7 +81,7 @@ public class TsFileWriteWithTablet {
private static void writeWithTablet(
TsFileWriter tsFileWriter,
String deviceId,
- List<MeasurementSchema> schemas,
+ List<IMeasurementSchema> schemas,
long rowNum,
long startTime,
long startValue)
diff --git a/java/pom.xml b/java/pom.xml
index fe5ed453..24b2cb59 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -24,16 +24,17 @@
<parent>
<groupId>org.apache.tsfile</groupId>
<artifactId>tsfile-parent</artifactId>
- <version>1.0.1-a6fb416-SNAPSHOT</version>
+ <version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>tsfile-java</artifactId>
- <version>1.0.1-a6fb416-SNAPSHOT</version>
+ <version>1.2.0-SNAPSHOT</version>
<packaging>pom</packaging>
<name>TsFile: Java</name>
<modules>
<module>common</module>
<module>tsfile</module>
<module>examples</module>
+ <module>tools</module>
</modules>
<dependencyManagement>
<dependencies>
diff --git a/java/tools/README-zh.md b/java/tools/README-zh.md
new file mode 100644
index 00000000..e0b8d7d3
--- /dev/null
+++ b/java/tools/README-zh.md
@@ -0,0 +1,126 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+
+[English](./README.md) | [中文](./README-zh.md)
+# TsFile Tools 手册
+## 简介
+
+## 开发
+
+### 前置条件
+
+构建 Java 版的 TsFile Tools，必须要安装以下依赖:
+
+1. Java >= 1.8 (1.8, 11 到 17 都经过验证. 请确保设置了环境变量).
+2. Maven >= 3.6 (如果要从源代码编译TsFile).
+
+
+### 使用 maven 构建
+
+```
+mvn clean package -P with-java -DskipTests
+```
+
+### 安装到本地机器
+
+```
+mvn install -P with-java -DskipTests
+```
+
+## schema 定义
+
+| 参数 | 说明 | 是否必填 | 默认值 |
+|------------|--------------------------|------|------|
+| table_name | 表名 | 是 | |
+| time_precision | 时间精度（可选值有：ms/us/ns） | 否 | ms |
+| has_header | 是否包含表头 (可选值有：true/false) | 否 | true |
+| separator | 行内分隔符（可选值有：, /tab/ ;） | 否 | , |
+| null_format | 空值 | 否 | |
+| id_columns | 主键列，支持cvs中不存在的列做为层级 | 否 | |
+| time_column | 时间列 | 是 | |
+| csv_columns | 按照顺序与csv列一一对应 | 是 | |
+
+说明：
+
+id_columns 按照顺序进行设置值，支持csv 文件中不存在的列作为层级
+例如csv 只有a,b,c,d,time五列则
+id_columns
+a1 default aa
+a
+其中a1 不在csv列，为虚拟列，默认值为aa
+
+csv_columns 之后的内容为值列的定义，每一行的第一个字段为在tsfile中的测点名，第二个字段为类型
+当csv中某一列不需要写入 tsfile时，可以设置为 SKIP
+例：
+csv_columns
+地区 TEXT，
+厂号 TEXT,
+设备号 TEXT,
+SKIP,
+SKIP,
+时间 INT64,
+温度 FLOAT,
+排量 DOUBLE,
+
+### 数据示例
+csv 文件内容
+```
+地区, 厂号, 设备号, 型号, 维修周期, 时间, 温度, 排量
+河北, 1001, 1, 10, 1, 1, 80.0, 1000.0
+河北, 1001, 1, 10, 1, 4, 80.0, 1000.0
+河北, 1002, 7, 5, 2, 1, 90.0, 1200.0
+```
+schema 定义
+
+```
+table_name=root.db1
+time_precision=ms
+has_header=true
+separator=,
+null_format=\N
+
+
+id_columns
+集团 DEFAULT 大唐
+地区
+厂号
+设备号
+
+time_column=时间
+
+csv_columns
+地区 TEXT，
+厂号 TEXT,
+设备号 TEXT,
+SKIP,
+SKIP,
+时间 INT64,
+温度 FLOAT,
+排量 DOUBLE,
+```
+## 命令
+
+```
+csv2tsfile.sh --source ./xxx/xxx --target /xxx/xxx --fail_dir /xxx/xxx
+csv2tsfile.bat --source ./xxx/xxx --target /xxx/xxx --fail_dir /xxx/xxx
+```
+
+
diff --git a/java/tools/README.md b/java/tools/README.md
new file mode 100644
index 00000000..97858fde
--- /dev/null
+++ b/java/tools/README.md
@@ -0,0 +1,127 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+
+[English](./README.md) | [Chinese](./README-zh.md)
+# TsFile Tools Manual
+## Introduction
+
+## Development
+
+### Prerequisites
+
+To build the Java version of TsFile Tools, you must have the following dependencies installed:
+
+1. Java >= 1.8 (1.8, 11 to 17 are verified. Make sure the environment variable is set).
+2. Maven >= 3.6 (if you are compiling TsFile from source).
+
+### Build with Maven
+
+```sh
+mvn clean package -P with-java -DskipTests
+```
+
+### Install to local machine
+
+```
+mvn install -P with-java -DskipTests
+```
+
+## schema 定义
+| Parameter | Description | Required | Default |
+|----------------|--------------------------|----------|------|
+| table_name | Table name | Yes | |
+| time_precision | Time precision (options: ms/us/ns) | No | ms |
+| has_header | Whether it contains a header (options: true/false) | No | true |
+| separator | Delimiter (options: , /tab/ ;) | No | , |
+| null_format | Null value | No | |
+| id_columns | Primary key columns, supports columns not in the CSV as hierarchy | No | |
+| time_column | Time column | Yes | |
+| csv_columns | Corresponding columns in the CSV in order | Yes | |
+
+Explanation:
+
+The "id_columns" sets values in order and supports using columns that do not exist in the CSV file as levels.
+For example, if the CSV file has only five columns: "a", "b", "c", "d", and "time",
+id_columns
+a1 default aa
+a
+Among them, a1 is not in the CSV column and is a virtual column with a default value of aa
+
+The content after csv_columns is the definition of the value column, with the first field in each row being the measurement point name in tsfile and the second field being the type
+When a column in CSV does not need to be written to tsfile, it can be set to SKIP.
+
+Example:
+csv_columns
+Region TEXT,
+Factory Number TEXT,
+Device Number TEXT,
+SKIP,
+SKIP,
+Time INT64,
+Temperature FLOAT,
+Emission DOUBLE,
+
+Data Example
+CSV file content:
+
+### sample data
+
+CSV file content:
+```
+Region,FactoryNumber,DeviceNumber,Model,MaintenanceCycle,Time,Temperature,Emission
+hebei,1001, 1,10,1,1,80.0,1000.0
+hebei,1001,1,10,1,4,80.0,1000.0
+hebei,1002,7,5,2,1,90.0,1200.0
+```
+Schema definition
+
+```
+table_name=root.db1
+time_precision=ms
+has_header=true
+separator=,
+null_format=\N
+
+
+id_columns
+Group DEFAULT Datang
+Region
+FactoryNumber
+DeviceNumber
+
+time_column=Time
+
+csv_columns
+RegionTEXT,
+FactoryNumber TEXT,
+DeviceNumber TEXT,
+SKIP,
+SKIP,
+Time INT64,
+Temperature FLOAT,
+Emission DOUBLE,
+```
+## Commands
+
+```
+csv2tsfile.sh --source ./xxx/xxx --target /xxx/xxx --fail_dir /xxx/xxx
+csv2tsfile.bat --source ./xxx/xxx --target /xxx/xxx --fail_dir /xxx/xxx
+```
diff --git a/java/tools/pom.xml b/java/tools/pom.xml
new file mode 100644
index 00000000..d46e695c
--- /dev/null
+++ b/java/tools/pom.xml
@@ -0,0 +1,92 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.tsfile</groupId>
+ <artifactId>tsfile-java</artifactId>
+ <version>1.2.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>tools</artifactId>
+ <name>TsFile: Java: Tools</name>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.tsfile</groupId>
+ <artifactId>common</artifactId>
+ <version>1.2.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ <version>1.5.0</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tsfile</groupId>
+ <artifactId>tsfile</artifactId>
+ <version>1.2.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <executions>
+ <!-- Package binaries-->
+ <execution>
+ <id>server-assembly</id>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <phase>package</phase>
+ <configuration>
+ <descriptors>
+ <descriptor>src/assembly/tools.xml</descriptor>
+ </descriptors>
+ <appendAssemblyId>false</appendAssemblyId>
+ <archive>
+ <manifest>
+ <mainClass>org.apache.tsfile.tools.TsFileTool</mainClass>
+ <addDefaultImplementationEntries>true</addDefaultImplementationEntries>
+ <addDefaultSpecificationEntries>true</addDefaultSpecificationEntries>
+ </manifest>
+ </archive>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/java/tools/src/assembly/resources/conf/logback-cvs2tsfile.xml b/java/tools/src/assembly/resources/conf/logback-cvs2tsfile.xml
new file mode 100644
index 00000000..d4d2ed8f
--- /dev/null
+++ b/java/tools/src/assembly/resources/conf/logback-cvs2tsfile.xml
@@ -0,0 +1,49 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<configuration scan="true" scanPeriod="60 seconds">
+ <appender class="ch.qos.logback.core.ConsoleAppender" name="stdout">
+ <Target>System.out</Target>
+ <encoder>
+ <pattern>%d [%t] %-5p %C{25}:%L - %m %n</pattern>
+ <charset>utf-8</charset>
+ </encoder>
+ <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+ <level>ERROR</level>
+ </filter>
+ </appender>
+ <appender class="ch.qos.logback.core.rolling.RollingFileAppender" name="tools">
+ <Encoding>UTF-8</Encoding>
+ <file>${TSFILE_HOME}/logs/log_tools.log</file>
+ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+ <fileNamePattern>${TSFILE_HOME}/logs/log-tools-%d{yyyyMMdd}.log.gz</fileNamePattern>
+ <maxHistory>30</maxHistory>
+ </rollingPolicy>
+ <append>true</append>
+ <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
+ <pattern>%d [%t] %-5p %C{25}:%L - %m %n</pattern>
+ <charset>utf-8</charset>
+ </encoder>
+ <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+ <level>INFO</level>
+ </filter>
+ </appender>
+ <root level="all">
+ <appender-ref ref="stdout"/>
+ <appender-ref ref="tools"/>
+ </root>
+</configuration>
diff --git a/java/tools/src/assembly/resources/tools/csv2tsfile.bat b/java/tools/src/assembly/resources/tools/csv2tsfile.bat
new file mode 100644
index 00000000..d8bc1dff
--- /dev/null
+++ b/java/tools/src/assembly/resources/tools/csv2tsfile.bat
@@ -0,0 +1,50 @@
+@REM
+@REM Licensed to the Apache Software Foundation (ASF) under one
+@REM or more contributor license agreements. See the NOTICE file
+@REM distributed with this work for additional information
+@REM regarding copyright ownership. The ASF licenses this file
+@REM to you under the Apache License, Version 2.0 (the
+@REM "License"); you may not use this file except in compliance
+@REM with the License. You may obtain a copy of the License at
+@REM
+@REM http://www.apache.org/licenses/LICENSE-2.0
+@REM
+@REM Unless required by applicable law or agreed to in writing,
+@REM software distributed under the License is distributed on an
+@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@REM KIND, either express or implied. See the License for the
+@REM specific language governing permissions and limitations
+@REM under the License.
+@REM
+
+@echo off
+setlocal enabledelayedexpansion
+
+if "%OS%" == "Windows_NT" setlocal
+
+pushd %~dp0..
+if NOT DEFINED TSFILE_HOME set TSFILE_HOME=%CD%
+popd
+
+set JAVA_OPTS=-ea^
+ -DTSFILE_HOME="%TSFILE_HOME%"
+
+if NOT DEFINED JAVA_HOME goto :err
+
+echo ------------------------------------------
+echo Starting Csv to TsFile Script
+echo ------------------------------------------
+
+set CLASSPATH="%TSFILE_HOME%\lib\*"
+if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.tsfile.tools.TsFileTool
+
+set TSFILE_CONF=%TSFILE_HOME%\conf
+set "tsfile_params=-Dlogback.configurationFile=!IOTDB_CLI_CONF!\logback-cvs2tsfile.xml"
+start /B "" cmd /C "("%JAVA_HOME%\bin\java" -DTSFILE_HOME=!TSFILE_HOME! !tsfile_params! !JAVA_OPTS! -cp !CLASSPATH! !MAIN_CLASS! %*) > nul 2>&1"
+exit /b
+
+
+:err
+echo JAVA_HOME environment variable must be set!
+set ret_code=1
+exit /b
\ No newline at end of file
diff --git a/java/tools/src/assembly/resources/tools/csv2tsfile.sh b/java/tools/src/assembly/resources/tools/csv2tsfile.sh
new file mode 100644
index 00000000..972c9160
--- /dev/null
+++ b/java/tools/src/assembly/resources/tools/csv2tsfile.sh
@@ -0,0 +1,53 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+echo ------------------------------------------
+echo Starting Csv to TsFile Script
+echo ------------------------------------------
+
+if [ -z "${TSFILE_HOME}" ]; then
+ export TSFILE_HOME="$(cd "`dirname "$0"`"/..; pwd)"
+fi
+
+if [ -n "$JAVA_HOME" ]; then
+ for java in "$JAVA_HOME"/bin/amd64/java "$JAVA_HOME"/bin/java; do
+ if [ -x "$java" ]; then
+ JAVA="$java"
+ break
+ fi
+ done
+else
+ JAVA=java
+fi
+
+if [ -z $JAVA ] ; then
+ echo Unable to find java executable. Check JAVA_HOME and PATH environment variables. > /dev/stderr
+ exit 1;
+fi
+
+
+CLASSPATH=${TSFILE_HOME}/lib/*
+
+MAIN_CLASS=org.apache.tsfile.tools.TsFileTool
+
+TSFILE_CONF=${TSFILE_HOME}/conf
+tsfile_params="-Dlogback.configurationFile=${TSFILE_CONF}/logback-cvs2tsfile.xml"
+
+exec "$JAVA" -DTSFILE_HOME=${TSFILE_HOME} $tsfile_params -cp "$CLASSPATH" "$MAIN_CLASS" "$@"
\ No newline at end of file
diff --git a/java/tools/src/assembly/tools.xml b/java/tools/src/assembly/tools.xml
new file mode 100644
index 00000000..ccd96051
--- /dev/null
+++ b/java/tools/src/assembly/tools.xml
@@ -0,0 +1,54 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<assembly>
+ <id>tools</id>
+ <formats>
+ <format>dir</format>
+ <format>zip</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <dependencySets>
+ <dependencySet>
+ <outputDirectory>lib</outputDirectory>
+ </dependencySet>
+ </dependencySets>
+ <fileSets>
+ <fileSet>
+ <directory>src/assembly/resources</directory>
+ <outputDirectory>${file.separator}</outputDirectory>
+ </fileSet>
+ </fileSets>
+ <files>
+ <file>
+ <source>${maven.multiModuleProjectDirectory}/java/tools/src/assembly/resources/conf/logback-cvs2tsfile.xml</source>
+ <destName>conf/logback-cvs2tsfile.xml</destName>
+ </file>
+ <file>
+ <source>${maven.multiModuleProjectDirectory}/java/tools/src/assembly/resources/tools/csv2tsfile.sh</source>
+ <destName>tools/csv2tsfile.sh</destName>
+ </file>
+ <file>
+ <source>${maven.multiModuleProjectDirectory}/java/tools/src/assembly/resources/tools/csv2tsfile.bat</source>
+ <destName>tools/csv2tsfile.bat</destName>
+ </file>
+ </files>
+</assembly>
diff --git a/java/tools/src/main/java/org/apache/tsfile/tools/DateTimeUtils.java b/java/tools/src/main/java/org/apache/tsfile/tools/DateTimeUtils.java
new file mode 100644
index 00000000..04452d3f
--- /dev/null
+++ b/java/tools/src/main/java/org/apache/tsfile/tools/DateTimeUtils.java
@@ -0,0 +1,514 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tsfile.tools;
+
+import java.time.DateTimeException;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.format.DateTimeParseException;
+import java.time.format.SignStyle;
+import java.time.temporal.ChronoField;
+
+public class DateTimeUtils {
+
+ private DateTimeUtils() {
+ // forbidding instantiation
+ }
+
+ public static final DateTimeFormatter ISO_LOCAL_DATE_WIDTH_1_2;
+
+ static {
+ ISO_LOCAL_DATE_WIDTH_1_2 =
+ new DateTimeFormatterBuilder()
+ .appendValue(ChronoField.YEAR, 4, 19, SignStyle.NEVER)
+ .appendLiteral('-')
+ .appendValue(ChronoField.MONTH_OF_YEAR, 1, 2, SignStyle.NEVER)
+ .appendLiteral('-')
+ .appendValue(ChronoField.DAY_OF_MONTH, 1, 2, SignStyle.NEVER)
+ .toFormatter();
+ }
+
+ /** such as '2011/12/03'. */
+ public static final DateTimeFormatter ISO_LOCAL_DATE_WITH_SLASH;
+
+ static {
+ ISO_LOCAL_DATE_WITH_SLASH =
+ new DateTimeFormatterBuilder()
+ .appendValue(ChronoField.YEAR, 4, 19, SignStyle.NEVER)
+ .appendLiteral('/')
+ .appendValue(ChronoField.MONTH_OF_YEAR, 1, 2, SignStyle.NEVER)
+ .appendLiteral('/')
+ .appendValue(ChronoField.DAY_OF_MONTH, 1, 2, SignStyle.NEVER)
+ .toFormatter();
+ }
+
+ /** such as '2011.12.03'. */
+ public static final DateTimeFormatter ISO_LOCAL_DATE_WITH_DOT;
+
+ static {
+ ISO_LOCAL_DATE_WITH_DOT =
+ new DateTimeFormatterBuilder()
+ .appendValue(ChronoField.YEAR, 4, 19, SignStyle.NEVER)
+ .appendLiteral('.')
+ .appendValue(ChronoField.MONTH_OF_YEAR, 1, 2, SignStyle.NEVER)
+ .appendLiteral('.')
+ .appendValue(ChronoField.DAY_OF_MONTH, 1, 2, SignStyle.NEVER)
+ .toFormatter();
+ }
+
+ /** such as '10:15:30' or '10:15:30.123'. */
+ public static final DateTimeFormatter ISO_LOCAL_TIME_WITH_MS;
+
+ static {
+ ISO_LOCAL_TIME_WITH_MS =
+ new DateTimeFormatterBuilder()
+ .appendValue(ChronoField.HOUR_OF_DAY, 2)
+ .appendLiteral(':')
+ .appendValue(ChronoField.MINUTE_OF_HOUR, 2)
+ .appendLiteral(':')
+ .appendValue(ChronoField.SECOND_OF_MINUTE, 2)
+ .optionalStart()
+ .appendFraction(ChronoField.MILLI_OF_SECOND, 0, 3, true)
+ .toFormatter();
+ }
+
+ /** such as '10:15:30' or '10:15:30.123456'. */
+ public static final DateTimeFormatter ISO_LOCAL_TIME_WITH_US;
+
+ static {
+ ISO_LOCAL_TIME_WITH_US =
+ new DateTimeFormatterBuilder()
+ .appendValue(ChronoField.HOUR_OF_DAY, 2)
+ .appendLiteral(':')
+ .appendValue(ChronoField.MINUTE_OF_HOUR, 2)
+ .appendLiteral(':')
+ .appendValue(ChronoField.SECOND_OF_MINUTE, 2)
+ .optionalStart()
+ .appendFraction(ChronoField.MICRO_OF_SECOND, 0, 6, true)
+ .toFormatter();
+ }
+
+ /** such as '10:15:30' or '10:15:30.123456789'. */
+ public static final DateTimeFormatter ISO_LOCAL_TIME_WITH_NS;
+
+ static {
+ ISO_LOCAL_TIME_WITH_NS =
+ new DateTimeFormatterBuilder()
+ .appendValue(ChronoField.HOUR_OF_DAY, 2)
+ .appendLiteral(':')
+ .appendValue(ChronoField.MINUTE_OF_HOUR, 2)
+ .appendLiteral(':')
+ .appendValue(ChronoField.SECOND_OF_MINUTE, 2)
+ .optionalStart()
+ .appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true)
+ .optionalEnd()
+ .toFormatter();
+ }
+
+ /** such as '2011-12-03T10:15:30+01:00' or '2011-12-03T10:15:30.123+01:00'. */
+ public static final DateTimeFormatter ISO_OFFSET_DATE_TIME_WITH_MS;
+
+ static {
+ ISO_OFFSET_DATE_TIME_WITH_MS =
+ new DateTimeFormatterBuilder()
+ .parseCaseInsensitive()
+ .append(ISO_LOCAL_DATE_WIDTH_1_2)
+ .appendLiteral('T')
+ .append(ISO_LOCAL_TIME_WITH_MS)
+ .appendOffsetId()
+ .toFormatter();
+ }
+
+ /** such as '2011-12-03T10:15:30+01:00' or '2011-12-03T10:15:30.123456+01:00'. */
+ public static final DateTimeFormatter ISO_OFFSET_DATE_TIME_WITH_US;
+
+ static {
+ ISO_OFFSET_DATE_TIME_WITH_US =
+ new DateTimeFormatterBuilder()
+ .parseCaseInsensitive()
+ .append(ISO_LOCAL_DATE_WIDTH_1_2)
+ .appendLiteral('T')
+ .append(ISO_LOCAL_TIME_WITH_US)
+ .appendOffsetId()
+ .toFormatter();
+ }
+
+ /** such as '2011-12-03T10:15:30+01:00' or '2011-12-03T10:15:30.123456789+01:00'. */
+ public static final DateTimeFormatter ISO_OFFSET_DATE_TIME_WITH_NS;
+
+ static {
+ ISO_OFFSET_DATE_TIME_WITH_NS =
+ new DateTimeFormatterBuilder()
+ .parseCaseInsensitive()
+ .append(ISO_LOCAL_DATE_WIDTH_1_2)
+ .appendLiteral('T')
+ .append(ISO_LOCAL_TIME_WITH_NS)
+ .appendOffsetId()
+ .toFormatter();
+ }
+
+ /** such as '2011/12/03T10:15:30+01:00' or '2011/12/03T10:15:30.123+01:00'. */
+ public static final DateTimeFormatter ISO_OFFSET_DATE_TIME_WITH_SLASH;
+
+ static {
+ ISO_OFFSET_DATE_TIME_WITH_SLASH =
+ new DateTimeFormatterBuilder()
+ .parseCaseInsensitive()
+ .append(ISO_LOCAL_DATE_WITH_SLASH)
+ .appendLiteral('T')
+ .append(ISO_LOCAL_TIME_WITH_MS)
+ .appendOffsetId()
+ .toFormatter();
+ }
+
+ /** such as '2011/12/03T10:15:30+01:00' or '2011/12/03T10:15:30.123456+01:00'. */
+ public static final DateTimeFormatter ISO_OFFSET_DATE_TIME_WITH_SLASH_US;
+
+ static {
+ ISO_OFFSET_DATE_TIME_WITH_SLASH_US =
+ new DateTimeFormatterBuilder()
+ .parseCaseInsensitive()
+ .append(ISO_LOCAL_DATE_WITH_SLASH)
+ .appendLiteral('T')
+ .append(ISO_LOCAL_TIME_WITH_US)
+ .appendOffsetId()
+ .toFormatter();
+ }
+
+ /** such as '2011/12/03T10:15:30+01:00' or '2011/12/03T10:15:30.123456789+01:00'. */
+ public static final DateTimeFormatter ISO_OFFSET_DATE_TIME_WITH_SLASH_NS;
+
+ static {
+ ISO_OFFSET_DATE_TIME_WITH_SLASH_NS =
+ new DateTimeFormatterBuilder()
+ .parseCaseInsensitive()
+ .append(ISO_LOCAL_DATE_WITH_SLASH)
+ .appendLiteral('T')
+ .append(ISO_LOCAL_TIME_WITH_NS)
+ .appendOffsetId()
+ .toFormatter();
+ }
+
+ /** such as '2011.12.03T10:15:30+01:00' or '2011.12.03T10:15:30.123+01:00'. */
+ public static final DateTimeFormatter ISO_OFFSET_DATE_TIME_WITH_DOT;
+
+ static {
+ ISO_OFFSET_DATE_TIME_WITH_DOT =
+ new DateTimeFormatterBuilder()
+ .parseCaseInsensitive()
+ .append(ISO_LOCAL_DATE_WITH_DOT)
+ .appendLiteral('T')
+ .append(ISO_LOCAL_TIME_WITH_MS)
+ .appendOffsetId()
+ .toFormatter();
+ }
+
+ /** such as '2011.12.03T10:15:30+01:00' or '2011.12.03T10:15:30.123456+01:00'. */
+ public static final DateTimeFormatter ISO_OFFSET_DATE_TIME_WITH_DOT_US;
+
+ static {
+ ISO_OFFSET_DATE_TIME_WITH_DOT_US =
+ new DateTimeFormatterBuilder()
+ .parseCaseInsensitive()
+ .append(ISO_LOCAL_DATE_WITH_DOT)
+ .appendLiteral('T')
+ .append(ISO_LOCAL_TIME_WITH_US)
+ .appendOffsetId()
+ .toFormatter();
+ }
+
+ /** such as '2011.12.03T10:15:30+01:00' or '2011.12.03T10:15:30.123456789+01:00'. */
+ public static final DateTimeFormatter ISO_OFFSET_DATE_TIME_WITH_DOT_NS;
+
+ static {
+ ISO_OFFSET_DATE_TIME_WITH_DOT_NS =
+ new DateTimeFormatterBuilder()
+ .parseCaseInsensitive()
+ .append(ISO_LOCAL_DATE_WITH_DOT)
+ .appendLiteral('T')
+ .append(ISO_LOCAL_TIME_WITH_NS)
+ .appendOffsetId()
+ .toFormatter();
+ }
+
+ /** such as '2011-12-03 10:15:30+01:00' or '2011-12-03 10:15:30.123+01:00'. */
+ public static final DateTimeFormatter ISO_OFFSET_DATE_TIME_WITH_SPACE;
+
+ static {
+ ISO_OFFSET_DATE_TIME_WITH_SPACE =
+ new DateTimeFormatterBuilder()
+ .parseCaseInsensitive()
+ .append(DateTimeFormatter.ISO_LOCAL_DATE)
+ .appendLiteral(' ')
+ .append(ISO_LOCAL_TIME_WITH_MS)
+ .appendOffsetId()
+ .toFormatter();
+ }
+
+ /** such as '2011-12-03 10:15:30+01:00' or '2011-12-03 10:15:30.123456+01:00'. */
+ public static final DateTimeFormatter ISO_OFFSET_DATE_TIME_WITH_SPACE_US;
+
+ static {
+ ISO_OFFSET_DATE_TIME_WITH_SPACE_US =
+ new DateTimeFormatterBuilder()
+ .parseCaseInsensitive()
+ .append(DateTimeFormatter.ISO_LOCAL_DATE)
+ .appendLiteral(' ')
+ .append(ISO_LOCAL_TIME_WITH_US)
+ .appendOffsetId()
+ .toFormatter();
+ }
+
+ /** such as '2011-12-03 10:15:30+01:00' or '2011-12-03 10:15:30.123456789+01:00'. */
+ public static final DateTimeFormatter ISO_OFFSET_DATE_TIME_WITH_SPACE_NS;
+
+ static {
+ ISO_OFFSET_DATE_TIME_WITH_SPACE_NS =
+ new DateTimeFormatterBuilder()
+ .parseCaseInsensitive()
+ .append(DateTimeFormatter.ISO_LOCAL_DATE)
+ .appendLiteral(' ')
+ .append(ISO_LOCAL_TIME_WITH_NS)
+ .appendOffsetId()
+ .toFormatter();
+ }
+
+ /** such as '2011/12/03 10:15:30+01:00' or '2011/12/03 10:15:30.123+01:00'. */
+ public static final DateTimeFormatter ISO_OFFSET_DATE_TIME_WITH_SLASH_WITH_SPACE;
+
+ static {
+ ISO_OFFSET_DATE_TIME_WITH_SLASH_WITH_SPACE =
+ new DateTimeFormatterBuilder()
+ .parseCaseInsensitive()
+ .append(ISO_LOCAL_DATE_WITH_SLASH)
+ .appendLiteral(' ')
+ .append(ISO_LOCAL_TIME_WITH_MS)
+ .appendOffsetId()
+ .toFormatter();
+ }
+
+ /** such as '2011/12/03 10:15:30+01:00' or '2011/12/03 10:15:30.123456+01:00'. */
+ public static final DateTimeFormatter ISO_OFFSET_DATE_TIME_WITH_SLASH_WITH_SPACE_US;
+
+ static {
+ ISO_OFFSET_DATE_TIME_WITH_SLASH_WITH_SPACE_US =
+ new DateTimeFormatterBuilder()
+ .parseCaseInsensitive()
+ .append(ISO_LOCAL_DATE_WITH_SLASH)
+ .appendLiteral(' ')
+ .append(ISO_LOCAL_TIME_WITH_US)
+ .appendOffsetId()
+ .toFormatter();
+ }
+
+ /** such as '2011/12/03 10:15:30+01:00' or '2011/12/03 10:15:30.123456789+01:00'. */
+ public static final DateTimeFormatter ISO_OFFSET_DATE_TIME_WITH_SLASH_WITH_SPACE_NS;
+
+ static {
+ ISO_OFFSET_DATE_TIME_WITH_SLASH_WITH_SPACE_NS =
+ new DateTimeFormatterBuilder()
+ .parseCaseInsensitive()
+ .append(ISO_LOCAL_DATE_WITH_SLASH)
+ .appendLiteral(' ')
+ .append(ISO_LOCAL_TIME_WITH_NS)
+ .appendOffsetId()
+ .toFormatter();
+ }
+
+ /** such as '2011.12.03 10:15:30+01:00' or '2011.12.03 10:15:30.123+01:00'. */
+ public static final DateTimeFormatter ISO_OFFSET_DATE_TIME_WITH_DOT_WITH_SPACE;
+
+ static {
+ ISO_OFFSET_DATE_TIME_WITH_DOT_WITH_SPACE =
+ new DateTimeFormatterBuilder()
+ .parseCaseInsensitive()
+ .append(ISO_LOCAL_DATE_WITH_DOT)
+ .appendLiteral(' ')
+ .append(ISO_LOCAL_TIME_WITH_MS)
+ .appendOffsetId()
+ .toFormatter();
+ }
+
+ /** such as '2011.12.03 10:15:30+01:00' or '2011.12.03 10:15:30.123456+01:00'. */
+ public static final DateTimeFormatter ISO_OFFSET_DATE_TIME_WITH_DOT_WITH_SPACE_US;
+
+ static {
+ ISO_OFFSET_DATE_TIME_WITH_DOT_WITH_SPACE_US =
+ new DateTimeFormatterBuilder()
+ .parseCaseInsensitive()
+ .append(ISO_LOCAL_DATE_WITH_DOT)
+ .appendLiteral(' ')
+ .append(ISO_LOCAL_TIME_WITH_US)
+ .appendOffsetId()
+ .toFormatter();
+ }
+
+ /** such as '2011.12.03 10:15:30+01:00' or '2011.12.03 10:15:30.123456789+01:00'. */
+ public static final DateTimeFormatter ISO_OFFSET_DATE_TIME_WITH_DOT_WITH_SPACE_NS;
+
+ static {
+ ISO_OFFSET_DATE_TIME_WITH_DOT_WITH_SPACE_NS =
+ new DateTimeFormatterBuilder()
+ .parseCaseInsensitive()
+ .append(ISO_LOCAL_DATE_WITH_DOT)
+ .appendLiteral(' ')
+ .append(ISO_LOCAL_TIME_WITH_NS)
+ .appendOffsetId()
+ .toFormatter();
+ }
+
+ public static final DateTimeFormatter formatter =
+ new DateTimeFormatterBuilder()
+ /**
+ * The ISO date-time formatter that formats or parses a date-time with an offset, such as
+ * '2011-12-03T10:15:30+01:00' or '2011-12-03T10:15:30.123+01:00'.
+ */
+ .appendOptional(ISO_OFFSET_DATE_TIME_WITH_MS)
+
+ /** such as '2011-12-03T10:15:30+01:00' or '2011-12-03T10:15:30.123456+01:00'. */
+ .appendOptional(ISO_OFFSET_DATE_TIME_WITH_US)
+
+ /** such as '2011-12-03T10:15:30+01:00' or '2011-12-03T10:15:30.123456789+01:00'. */
+ .appendOptional(ISO_OFFSET_DATE_TIME_WITH_NS)
+
+ /** such as '2011/12/03T10:15:30+01:00' or '2011/12/03T10:15:30.123+01:00'. */
+ .appendOptional(ISO_OFFSET_DATE_TIME_WITH_SLASH)
+
+ /** such as '2011/12/03T10:15:30+01:00' or '2011/12/03T10:15:30.123456+01:00'. */
+ .appendOptional(ISO_OFFSET_DATE_TIME_WITH_SLASH_US)
+
+ /** such as '2011/12/03T10:15:30+01:00' or '2011/12/03T10:15:30.123456789+01:00'. */
+ .appendOptional(ISO_OFFSET_DATE_TIME_WITH_SLASH_NS)
+
+ /** such as '2011.12.03T10:15:30+01:00' or '2011.12.03T10:15:30.123+01:00'. */
+ .appendOptional(ISO_OFFSET_DATE_TIME_WITH_DOT)
+
+ /** such as '2011.12.03T10:15:30+01:00' or '2011.12.03T10:15:30.123456+01:00'. */
+ .appendOptional(ISO_OFFSET_DATE_TIME_WITH_DOT_US)
+
+ /** such as '2011.12.03T10:15:30+01:00' or '2011.12.03T10:15:30.123456789+01:00'. */
+ .appendOptional(ISO_OFFSET_DATE_TIME_WITH_DOT_NS)
+
+ /** such as '2011-12-03 10:15:30+01:00' or '2011-12-03 10:15:30.123+01:00'. */
+ .appendOptional(ISO_OFFSET_DATE_TIME_WITH_SPACE)
+
+ /** such as '2011-12-03 10:15:30+01:00' or '2011-12-03 10:15:30.123456+01:00'. */
+ .appendOptional(ISO_OFFSET_DATE_TIME_WITH_SPACE_US)
+
+ /** such as '2011-12-03 10:15:30+01:00' or '2011-12-03 10:15:30.123456789+01:00'. */
+ .appendOptional(ISO_OFFSET_DATE_TIME_WITH_SPACE_NS)
+
+ /** such as '2011/12/03 10:15:30+01:00' or '2011/12/03 10:15:30.123+01:00'. */
+ .appendOptional(ISO_OFFSET_DATE_TIME_WITH_SLASH_WITH_SPACE)
+
+ /** such as '2011/12/03 10:15:30+01:00' or '2011/12/03 10:15:30.123456+01:00'. */
+ .appendOptional(ISO_OFFSET_DATE_TIME_WITH_SLASH_WITH_SPACE_US)
+
+ /** such as '2011/12/03 10:15:30+01:00' or '2011/12/03 10:15:30.123456789+01:00'. */
+ .appendOptional(ISO_OFFSET_DATE_TIME_WITH_SLASH_WITH_SPACE_NS)
+
+ /** such as '2011.12.03 10:15:30+01:00' or '2011.12.03 10:15:30.123+01:00'. */
+ .appendOptional(ISO_OFFSET_DATE_TIME_WITH_DOT_WITH_SPACE)
+
+ /** such as '2011.12.03 10:15:30+01:00' or '2011.12.03 10:15:30.123456+01:00'. */
+ .appendOptional(ISO_OFFSET_DATE_TIME_WITH_DOT_WITH_SPACE_US)
+
+ /** such as '2011.12.03 10:15:30+01:00' or '2011.12.03 10:15:30.123456789+01:00'. */
+ .appendOptional(ISO_OFFSET_DATE_TIME_WITH_DOT_WITH_SPACE_NS)
+ .toFormatter();
+
+ public static long convertTimestampOrDatetimeStrToLongWithDefaultZone(
+ String timeStr, String timestampPrecision) {
+ try {
+ return Long.parseLong(timeStr);
+ } catch (NumberFormatException e) {
+ return DateTimeUtils.convertDatetimeStrToLong(
+ timeStr, ZoneId.systemDefault(), timestampPrecision);
+ }
+ }
+
+ public static long convertDatetimeStrToLong(String str, ZoneId zoneId) {
+ return convertDatetimeStrToLong(str, toZoneOffset(zoneId), 0, "ms");
+ }
+
+ public static long convertDatetimeStrToLong(
+ String str, ZoneId zoneId, String timestampPrecision) {
+ return convertDatetimeStrToLong(str, toZoneOffset(zoneId), 0, timestampPrecision);
+ }
+
+ public static long getInstantWithPrecision(String str, String timestampPrecision) {
+ try {
+ ZonedDateTime zonedDateTime = ZonedDateTime.parse(str, formatter);
+ Instant instant = zonedDateTime.toInstant();
+ if ("us".equals(timestampPrecision)) {
+ if (instant.getEpochSecond() < 0 && instant.getNano() > 0) {
+ // adjustment can reduce the loss of the division
+ long millis = Math.multiplyExact(instant.getEpochSecond() + 1, 1000_000L);
+ long adjustment = instant.getNano() / 1000 - 1L;
+ return Math.addExact(millis, adjustment);
+ } else {
+ long millis = Math.multiplyExact(instant.getEpochSecond(), 1000_000L);
+ return Math.addExact(millis, instant.getNano() / 1000);
+ }
+ } else if ("ns".equals(timestampPrecision)) {
+ long millis = Math.multiplyExact(instant.getEpochSecond(), 1000_000_000L);
+ return Math.addExact(millis, instant.getNano());
+ }
+ return instant.toEpochMilli();
+ } catch (DateTimeParseException e) {
+ throw new RuntimeException(e.getMessage());
+ }
+ }
+
+ /** convert date time string to millisecond, microsecond or nanosecond. */
+ public static long convertDatetimeStrToLong(
+ String str, ZoneOffset offset, int depth, String timestampPrecision) {
+ if (depth >= 2) {
+ throw new DateTimeException(
+ String.format(
+ "Failed to convert %s to millisecond, zone offset is %s, "
+ + "please input like 2011-12-03T10:15:30 or 2011-12-03T10:15:30+01:00",
+ str, offset));
+ }
+ if (str.contains("Z")) {
+ return convertDatetimeStrToLong(
+ str.substring(0, str.indexOf('Z')) + "+00:00", offset, depth, timestampPrecision);
+ } else if (str.length() == 10) {
+ return convertDatetimeStrToLong(str + "T00:00:00", offset, depth, timestampPrecision);
+ } else if (str.length() - str.lastIndexOf('+') != 6
+ && str.length() - str.lastIndexOf('-') != 6) {
+ return convertDatetimeStrToLong(str + offset, offset, depth + 1, timestampPrecision);
+ } else if (str.contains("[") || str.contains("]")) {
+ throw new DateTimeException(
+ String.format(
+ "%s with [time-region] at end is not supported now, "
+ + "please input like 2011-12-03T10:15:30 or 2011-12-03T10:15:30+01:00",
+ str));
+ }
+ return getInstantWithPrecision(str, timestampPrecision);
+ }
+
+ public static ZoneOffset toZoneOffset(ZoneId zoneId) {
+ return zoneId.getRules().getOffset(Instant.now());
+ }
+}
diff --git a/java/tools/src/main/java/org/apache/tsfile/tools/SchemaParser.java b/java/tools/src/main/java/org/apache/tsfile/tools/SchemaParser.java
new file mode 100644
index 00000000..0bb9816e
--- /dev/null
+++ b/java/tools/src/main/java/org/apache/tsfile/tools/SchemaParser.java
@@ -0,0 +1,264 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tsfile.tools;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class SchemaParser {
+
+ public static class Schema {
+ String tableName;
+ String timePrecision;
+ boolean hasHeader = true;
+ String separator;
+ String nullFormat;
+ String timeColumn;
+ int timeColumnIndex = -1;
+ List<IDColumns> idColumns = new ArrayList<>();
+ List<Column> csvColumns = new ArrayList<>();
+
+ @Override
+ public String toString() {
+ return "Schema{"
+ + "tableName='"
+ + tableName
+ + '\''
+ + ", timePrecision='"
+ + timePrecision
+ + '\''
+ + ", hasHeader="
+ + hasHeader
+ + ", separator='"
+ + separator
+ + '\''
+ + ", nullFormat='"
+ + nullFormat
+ + '\''
+ + ", timeColumn='"
+ + timeColumn
+ + '\''
+ + ", idColumns="
+ + idColumns
+ + ", csvColumns="
+ + csvColumns
+ + '}';
+ }
+ }
+
+ public static class Column {
+ String name;
+ String type;
+
+ boolean isSkip;
+
+ public Column(String name, String type) {
+ this.name = name;
+ this.isSkip = false;
+ this.type = type;
+ }
+
+ public Column(String name) {
+ this.name = name;
+ this.isSkip = true;
+ }
+
+ @Override
+ public String toString() {
+ return "Column{"
+ + "name='"
+ + name
+ + '\''
+ + ", type='"
+ + type
+ + '\''
+ + ", isSkip="
+ + isSkip
+ + '}';
+ }
+ }
+
+ public static class IDColumns {
+ String name;
+ boolean isDefault;
+ String defaultValue;
+ int csvColumnIndex = -1;
+ boolean isExistCsvColumn;
+
+ public IDColumns(String name, boolean isDefault, String defaultValue) {
+ this.name = name;
+ this.isDefault = isDefault;
+ if (isDefault) {
+ this.defaultValue = defaultValue;
+ this.isExistCsvColumn = false;
+ }
+ }
+
+ public IDColumns(String name) {
+ this.name = name;
+ this.isDefault = false;
+ this.isExistCsvColumn = true;
+ }
+
+ @Override
+ public String toString() {
+ return "IDColumns{"
+ + "name='"
+ + name
+ + '\''
+ + ", isDefault="
+ + isDefault
+ + ", defaultValue='"
+ + defaultValue
+ + '\''
+ + ", isExistCsvColumn="
+ + isExistCsvColumn
+ + ", csvColumnIndex="
+ + csvColumnIndex
+ + '}';
+ }
+ }
+
+ public static Schema parseSchema(String filePath) throws IOException {
+ Schema schema = new Schema();
+ try (BufferedReader reader = new BufferedReader(new FileReader(filePath))) {
+ String line;
+ boolean readingIdColumns = false;
+ boolean readingCsvColumns = false;
+ int timeIndex = 0;
+ while ((line = reader.readLine()) != null) {
+ line = line.trim();
+ if (line.isEmpty() || line.startsWith("//")) {
+ continue;
+ }
+ if (line.startsWith("table_name=")) {
+ schema.tableName = extractValue(line);
+ } else if (line.startsWith("time_precision=")) {
+ schema.timePrecision = extractValue(line);
+ } else if (line.startsWith("has_header=")) {
+ schema.hasHeader = Boolean.parseBoolean(extractValue(line));
+ } else if (line.startsWith("separator=")) {
+ schema.separator = extractValue(line);
+ } else if (line.startsWith("null_format=")) {
+ schema.nullFormat = extractValue(line);
+ } else if (line.startsWith("time_column=")) {
+ schema.timeColumn = extractValue(line);
+ } else if (line.equals("id_columns")) {
+ readingIdColumns = true;
+ readingCsvColumns = false;
+ } else if (line.equals("csv_columns")) {
+ readingIdColumns = false;
+ readingCsvColumns = true;
+ } else if (readingIdColumns) {
+ parseIdColumns(line, schema);
+ } else if (readingCsvColumns) {
+ parseCsvColumns(line, schema, timeIndex);
+ timeIndex++;
+ }
+ }
+ addIdColumnsIndex(schema);
+ }
+ validateParams(schema);
+ if (schema.separator.equals("tab")) {
+ schema.separator = "\t";
+ }
+ return schema;
+ }
+
+ private static String extractValue(String line) {
+ int index = line.indexOf('=');
+ return line.substring(index + 1);
+ }
+
+ private static void parseIdColumns(String line, Schema schema) {
+ String[] parts = line.split(" ");
+ if (parts.length == 3) {
+ schema.idColumns.add(
+ new IDColumns(
+ parts[0].trim(), parts[1].trim().equalsIgnoreCase("DEFAULT"), parts[2].trim()));
+ } else if (parts.length == 1) {
+ schema.idColumns.add(new IDColumns(parts[0].trim()));
+ } else {
+ throw new IllegalArgumentException("The data format of id_columns is incorrect");
+ }
+ }
+
+ private static void addIdColumnsIndex(Schema schema) {
+ List<IDColumns> idColumnsList = schema.idColumns;
+ List<Column> columnList = schema.csvColumns;
+ for (IDColumns idColumn : idColumnsList) {
+ if (!idColumn.isDefault) {
+ for (int j = 0; j < columnList.size(); j++) {
+ if (columnList.get(j).name.equals(idColumn.name)) {
+ idColumn.csvColumnIndex = j;
+ break;
+ }
+ }
+ }
+ }
+ }
+
+ private static void parseCsvColumns(String line, Schema schema, int timeIndex) {
+ String[] parts = line.split(" ");
+ String columnName = parts[0].trim();
+
+ if (parts.length == 2) {
+ String dataType = parts[1].trim();
+ if (dataType.endsWith(",") || dataType.endsWith(";")) {
+ dataType = dataType.substring(0, dataType.length() - 1);
+ }
+ if (columnName.equals(schema.timeColumn)) {
+ schema.timeColumnIndex = timeIndex;
+ }
+ schema.csvColumns.add(new Column(columnName, dataType));
+ } else if (parts.length == 1) {
+ if (columnName.endsWith(",") || columnName.endsWith(";")) {
+ columnName = columnName.substring(0, columnName.length() - 1);
+ }
+ schema.csvColumns.add(new Column(columnName));
+ } else {
+ throw new IllegalArgumentException("The data format of csv_columns is incorrect");
+ }
+ }
+
+ private static void validateParams(SchemaParser.Schema schema) {
+ if (!schema.timePrecision.equals("us")
+ && !schema.timePrecision.equals("ms")
+ && !schema.timePrecision.equals("ns")) {
+ throw new IllegalArgumentException("timePrecision must be us,ms or ns");
+ }
+ if (!schema.separator.equals(",")
+ && !schema.separator.equals("tab")
+ && !schema.separator.equals(";")) {
+ throw new IllegalArgumentException("separator must be \",\", tab, or \";\"");
+ }
+ if (schema.timeColumnIndex < 0) {
+ throw new IllegalArgumentException("time_column is required");
+ }
+ if (schema.tableName.isEmpty()) {
+ throw new IllegalArgumentException("table_name is required");
+ }
+ if (schema.csvColumns.isEmpty()) {
+ throw new IllegalArgumentException("csv_columns is required");
+ }
+ }
+}
diff --git a/java/tools/src/main/java/org/apache/tsfile/tools/TsFileTool.java b/java/tools/src/main/java/org/apache/tsfile/tools/TsFileTool.java
new file mode 100644
index 00000000..0d5c1cfe
--- /dev/null
+++ b/java/tools/src/main/java/org/apache/tsfile/tools/TsFileTool.java
@@ -0,0 +1,498 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tsfile.tools;
+
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.TableSchema;
+import org.apache.tsfile.file.metadata.enums.CompressionType;
+import org.apache.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.tsfile.utils.Binary;
+import org.apache.tsfile.write.TsFileWriter;
+import org.apache.tsfile.write.record.Tablet;
+import org.apache.tsfile.write.schema.IMeasurementSchema;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.io.FilenameUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class TsFileTool {
+ private static int THREAD_COUNT = 8;
+ // Default value 256MB
+ private static long CHUNK_SIZE_BYTE = 1024 * 1024 * 256;
+ private static String outputDirectoryStr = "";
+ private static String inputDirectoryStr = "";
+ private static String failedDirectoryStr = "failed";
+ private static String schemaPathStr = "";
+
+ private static SchemaParser.Schema schema = null;
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(TsFileTool.class);
+
+ public static void main(String[] args) {
+ if (System.getenv("TSFILE_HOME") != null) {
+ System.setProperty("TSFILE_HOME", System.getenv("TSFILE_HOME"));
+ }
+ parseCommandLineParams(args);
+ if (!validateParams()) {
+ return;
+ }
+ createDir();
+ try {
+ schema = SchemaParser.parseSchema(schemaPathStr);
+ } catch (Exception e) {
+ LOGGER.error("Failed to parse schema file: " + schemaPathStr, e);
+ System.exit(1);
+ }
+ File inputDirectory = new File(inputDirectoryStr);
+
+ ExecutorService executor = Executors.newFixedThreadPool(THREAD_COUNT);
+
+ try {
+ processDirectory(inputDirectory, executor);
+ } finally {
+ executor.shutdown();
+ try {
+ executor.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS);
+ } catch (InterruptedException e) {
+ LOGGER.error("Failed to await termination", e);
+ }
+ }
+ }
+
+ private static TableSchema genTableSchema(
+ List<SchemaParser.IDColumns> idColumnList,
+ List<SchemaParser.Column> columnList,
+ String tableName,
+ Map<String, Object> defaultMap) {
+ List<IMeasurementSchema> measurementSchemas = new ArrayList<>();
+ List<Tablet.ColumnType> columnTypes = new ArrayList<>();
+ List<String> idSchemaList = new ArrayList<>();
+ for (SchemaParser.IDColumns idSchema : idColumnList) {
+ if (idSchema.isDefault) {
+ defaultMap.put(idSchema.name, idSchema.defaultValue);
+ }
+ idSchemaList.add(idSchema.name);
+ measurementSchemas.add(
+ new MeasurementSchema(
+ idSchema.name, TSDataType.TEXT, TSEncoding.PLAIN, CompressionType.UNCOMPRESSED));
+ columnTypes.add(Tablet.ColumnType.ID);
+ }
+ List<SchemaParser.Column> newColumnList = new ArrayList<>();
+
+ for (SchemaParser.Column column : columnList) {
+ if (!column.isSkip
+ && !idSchemaList.contains(column.name)
+ && !column.name.equals(schema.timeColumn)) {
+ newColumnList.add(column);
+ }
+ }
+
+ for (SchemaParser.Column column : newColumnList) {
+ measurementSchemas.add(
+ new MeasurementSchema(
+ column.name,
+ TSDataType.valueOf(column.type),
+ TSEncoding.PLAIN,
+ CompressionType.UNCOMPRESSED));
+ columnTypes.add(Tablet.ColumnType.MEASUREMENT);
+ }
+ return new TableSchema(tableName, measurementSchemas, columnTypes);
+ }
+
+ private static boolean writeTsFile(String fileName, List<String> lineList) {
+ final File tsFile = new File(outputDirectoryStr, fileName);
+ TsFileWriter writer = null;
+ try {
+ writer = new TsFileWriter(tsFile);
+ writer.setGenerateTableSchema(true);
+ Map<String, Object> defaultMap = new HashMap<>();
+ TableSchema tableSchema =
+ genTableSchema(schema.idColumns, schema.csvColumns, schema.tableName, defaultMap);
+ writer.registerTableSchema(tableSchema);
+ Tablet tablet = genTablet(tableSchema, lineList, defaultMap);
+ if (tablet != null) {
+ writer.writeTable(tablet);
+ return true;
+ } else {
+ return false;
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ LOGGER.error("Failed to write file: " + tsFile);
+ return false;
+ } finally {
+ if (writer != null) {
+ try {
+ writer.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+
+ private static Tablet genTablet(
+ TableSchema tableSchema, List<String> lineList, Map<String, Object> defaultMap) {
+ int num = lineList.size();
+ Tablet tablet =
+ new Tablet(
+ tableSchema.getTableName(),
+ tableSchema.getColumnSchemas(),
+ tableSchema.getColumnTypes(),
+ num);
+
+ Map<String, Integer> map = new HashMap<>();
+ for (int i = 0; i < schema.csvColumns.size(); i++) {
+ SchemaParser.Column column = schema.csvColumns.get(i);
+ map.put(column.name, i);
+ }
+ try {
+ for (int i = 0; i < num; i++) {
+ String line = lineList.get(i);
+ String[] lineArray = line.split(schema.separator);
+ long timestamp =
+ DateTimeUtils.convertTimestampOrDatetimeStrToLongWithDefaultZone(
+ lineArray[schema.timeColumnIndex], schema.timePrecision);
+
+ tablet.addTimestamp(i, timestamp);
+ List<IMeasurementSchema> columnSchemas = tableSchema.getColumnSchemas();
+ for (int j = 0; j < columnSchemas.size(); j++) {
+ IMeasurementSchema columnSchema = columnSchemas.get(j);
+ if (defaultMap.get(columnSchema.getMeasurementId()) != null) {
+ tablet.addValue(
+ columnSchema.getMeasurementId(),
+ i,
+ defaultMap.get(columnSchema.getMeasurementId()));
+ } else {
+ String value = lineArray[map.get(columnSchema.getMeasurementId())];
+ if (value.equals(schema.nullFormat)) {
+ value = null;
+ }
+ tablet.addValue(
+ columnSchema.getMeasurementId(),
+ i,
+ getValue(columnSchema.getType(), value, tableSchema.getColumnTypes().get(j)));
+ }
+ }
+ }
+ tablet.rowSize = num;
+ return tablet;
+ } catch (Exception e) {
+ LOGGER.error("Failed to parse csv file");
+ }
+ return null;
+ }
+
+ public static Object getValue(TSDataType dataType, String i, Tablet.ColumnType columnType) {
+ switch (dataType) {
+ case INT64:
+ return Long.valueOf(i);
+ case INT32:
+ return Integer.valueOf(i);
+ case BOOLEAN:
+ return Boolean.valueOf(i);
+ case TEXT:
+ if (columnType.equals(Tablet.ColumnType.MEASUREMENT)) {
+ return new Binary(String.valueOf(i), StandardCharsets.UTF_8);
+ } else {
+ return String.valueOf(i);
+ }
+ case FLOAT:
+ return Float.valueOf(i);
+ case DOUBLE:
+ return Double.valueOf(i);
+ default:
+ return i;
+ }
+ }
+
+ private static void processDirectory(File directory, ExecutorService executor) {
+ if (directory.isFile()) {
+ processFile(directory, executor);
+ } else {
+ File[] files = directory.listFiles();
+ if (files != null) {
+ for (File file : files) {
+ if (file.isDirectory()) {
+ processDirectory(file, executor);
+ } else if (file.isFile() && file.getName().endsWith(".csv")) {
+ processFile(file, executor);
+ }
+ }
+ }
+ }
+ }
+
+ private static void cpFile(String sourceFilePath, String targetDirectoryPath) {
+ try {
+ Files.createDirectories(Paths.get(targetDirectoryPath));
+ Path sourcePath = Paths.get(sourceFilePath);
+ Path targetPath = Paths.get(targetDirectoryPath, sourcePath.getFileName().toString());
+ Files.copy(sourcePath, targetPath, StandardCopyOption.REPLACE_EXISTING);
+ } catch (IOException e) {
+ LOGGER.error("Failed to copy file: " + sourceFilePath, e);
+ }
+ }
+
+ private static void processFile(File inputFile, ExecutorService executor) {
+ AtomicInteger fileCounter = new AtomicInteger(1);
+ String fileName = FilenameUtils.getBaseName(inputFile.getName());
+ String fileAbsolutePath = inputFile.getAbsolutePath();
+ try (BufferedReader reader =
+ new BufferedReader(
+ new InputStreamReader(
+ Files.newInputStream(inputFile.toPath()), StandardCharsets.UTF_8))) {
+ String line;
+ long currentChunkSize = 0;
+ int chunkLines = 0;
+ int index = 0;
+ List<String> lineList = new ArrayList<>();
+ boolean isSingleFile = true;
+ while ((line = reader.readLine()) != null) {
+ if (index == 0) {
+ if (schema.timeColumnIndex == -1) {
+ LOGGER.error(inputFile.getAbsolutePath() + " not found:" + schema.timeColumn);
+ cpFile(inputFile.getAbsolutePath(), failedDirectoryStr);
+ break;
+ }
+ String[] csvCloumns = line.split(schema.separator);
+ if (csvCloumns.length != schema.csvColumns.size()) {
+ LOGGER.error(
+ "The number of columns defined in the schema file is not equal to the number of columns in the csv file("
+ + inputFile.getAbsolutePath()
+ + ").");
+ cpFile(inputFile.getAbsolutePath(), failedDirectoryStr);
+ break;
+ }
+ }
+
+ if (schema.hasHeader && index == 0) {
+ index++;
+ continue;
+ }
+ index++;
+ byte[] lineBytes = line.getBytes(StandardCharsets.UTF_8);
+ long lineSize = lineBytes.length;
+ if (currentChunkSize + lineSize > CHUNK_SIZE_BYTE) {
+ isSingleFile = false;
+ if (chunkLines > 0) {
+ submitChunk(
+ lineList,
+ fileCounter.getAndIncrement(),
+ executor,
+ fileName,
+ isSingleFile,
+ fileAbsolutePath);
+ lineList = new ArrayList<>();
+ currentChunkSize = 0;
+ chunkLines = 0;
+ } else {
+ lineList.add(line);
+ submitChunk(
+ lineList,
+ fileCounter.getAndIncrement(),
+ executor,
+ fileName,
+ isSingleFile,
+ fileAbsolutePath);
+ lineList = new ArrayList<>();
+ currentChunkSize = 0;
+ chunkLines = 0;
+ }
+ }
+ lineList.add(line);
+ currentChunkSize += lineSize;
+ chunkLines++;
+ }
+ if (lineList.size() > 0) {
+ submitChunk(
+ lineList,
+ fileCounter.getAndIncrement(),
+ executor,
+ fileName,
+ isSingleFile,
+ fileAbsolutePath);
+ }
+
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ private static void submitChunk(
+ List<String> lineList,
+ int fileNumber,
+ ExecutorService executor,
+ String fileName,
+ boolean isSingleFile,
+ String fileAbsolutePath) {
+ executor.submit(
+ () -> {
+ boolean isSuccess;
+ if (isSingleFile) {
+ isSuccess = writeTsFile(fileName + ".tsfile", lineList);
+ } else {
+ isSuccess = writeTsFile(fileName + "_" + fileNumber + ".tsfile", lineList);
+ }
+ if (!isSuccess) {
+ cpFile(fileAbsolutePath, failedDirectoryStr);
+ }
+ });
+ }
+
+ private static void printHelp(Options options) {
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp("csv2tsfile.sh/csv2tsfile.bat", options);
+ }
+
+ private static void parseCommandLineParams(String[] args) {
+ Options options = new Options();
+ options.addOption("s", "source", true, "Input directory");
+ options.addOption("t", "target", true, "Output directory");
+ options.addOption("fd", "fail_dir", true, "Failed file directory");
+ options.addOption("b", "block_size", true, "Block size default value 256M");
+ options.addOption("tn", "thread_num", true, "Thread number");
+ options.addOption("schema", "schema", true, "Schema file path");
+ options.addOption("h", "help", false, "Show help");
+
+ try {
+ CommandLineParser parser = new DefaultParser();
+ CommandLine cmd = parser.parse(options, args);
+
+ if (cmd.hasOption("h")) {
+ printHelp(options);
+ return;
+ }
+
+ if (cmd.hasOption("s")) {
+ inputDirectoryStr = cmd.getOptionValue("s");
+ }
+ if (cmd.hasOption("t")) {
+ outputDirectoryStr = cmd.getOptionValue("t");
+ }
+ if (cmd.hasOption("fd")) {
+ failedDirectoryStr = cmd.getOptionValue("fd");
+ }
+ if (cmd.hasOption("b")) {
+ CHUNK_SIZE_BYTE = parseBlockSize(cmd.getOptionValue("b"));
+ }
+ if (cmd.hasOption("tn")) {
+ THREAD_COUNT = Integer.parseInt(cmd.getOptionValue("tn"));
+ }
+ if (cmd.hasOption("schema")) {
+ schemaPathStr = cmd.getOptionValue("schema");
+ }
+
+ if (failedDirectoryStr == null || failedDirectoryStr.equals("")) {
+ failedDirectoryStr = "failed";
+ }
+ } catch (ParseException e) {
+ e.printStackTrace();
+ }
+ }
+
+ private static long parseBlockSize(String blockSizeValue) {
+ long size;
+ blockSizeValue = blockSizeValue.toUpperCase();
+
+ if (blockSizeValue.endsWith("K")) {
+ size = Long.parseLong(blockSizeValue.substring(0, blockSizeValue.length() - 1)) * 1024;
+ } else if (blockSizeValue.endsWith("M")) {
+ size = Long.parseLong(blockSizeValue.substring(0, blockSizeValue.length() - 1)) * 1024 * 1024;
+ } else if (blockSizeValue.endsWith("G")) {
+ size =
+ Long.parseLong(blockSizeValue.substring(0, blockSizeValue.length() - 1))
+ * 1024
+ * 1024
+ * 1024;
+ } else if (blockSizeValue.endsWith("T") || blockSizeValue.endsWith("B")) {
+ throw new IllegalArgumentException("block_size only supports units of K, M, G, or numbers");
+ } else {
+ size = Long.parseLong(blockSizeValue);
+ }
+
+ return size;
+ }
+
+ private static void createDir() {
+ File targetDir = new File(outputDirectoryStr);
+ if (!targetDir.exists()) {
+ targetDir.mkdirs();
+ }
+ if (failedDirectoryStr != null) {
+ File failDirFile = new File(failedDirectoryStr);
+ if (!failDirFile.exists()) {
+ failDirFile.mkdirs();
+ }
+ }
+ }
+
+ private static boolean validateParams() {
+ if (inputDirectoryStr == null
+ || inputDirectoryStr.isEmpty()
+ || outputDirectoryStr == null
+ || outputDirectoryStr.isEmpty()
+ || schemaPathStr == null
+ || schemaPathStr.isEmpty()) {
+ LOGGER.error("Missing required parameters. Please provide --source, --target, and --schema.");
+ return false;
+ }
+ File sourceDir = new File(inputDirectoryStr);
+ if (!sourceDir.exists()) {
+ LOGGER.error("Source directory(" + sourceDir + ") does not exist.");
+ return false;
+ }
+ File schemaFile = new File(schemaPathStr);
+ if (!schemaFile.exists()) {
+ LOGGER.error("Schema file(" + schemaPathStr + ") does not exist.");
+ return false;
+ }
+ if (THREAD_COUNT <= 0) {
+ LOGGER.error("Invalid thread number. Thread number must be greater than 0.");
+ return false;
+ }
+
+ return true;
+ }
+}
diff --git a/java/tools/src/test/java/org/apache/tsfile/tools/TsfiletoolsTest.java b/java/tools/src/test/java/org/apache/tsfile/tools/TsfiletoolsTest.java
new file mode 100644
index 00000000..455ab1c5
--- /dev/null
+++ b/java/tools/src/test/java/org/apache/tsfile/tools/TsfiletoolsTest.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tsfile.tools;
+
+import org.apache.tsfile.read.TsFileSequenceReader;
+import org.apache.tsfile.read.common.block.TsBlock;
+import org.apache.tsfile.read.controller.CachedChunkLoaderImpl;
+import org.apache.tsfile.read.controller.MetadataQuerierByFileImpl;
+import org.apache.tsfile.read.query.executor.TableQueryExecutor;
+import org.apache.tsfile.read.reader.block.TsBlockReader;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class TsfiletoolsTest {
+ private final String testDir = "target" + File.separator + "csvTest";
+ private final String csvFile = testDir + File.separator + "data.csv";
+ private final String schemaFile = testDir + File.separator + "schemaFile.txt";
+
+ float[] tmpResult2 = new float[20];
+ float[] tmpResult3 = new float[20];
+ float[] tmpResult5 = new float[20];
+
+ @Before
+ public void setUp() {
+ new File(testDir).mkdirs();
+ genCsvFile(20);
+ genSchemaFile();
+ }
+
+ public void genSchemaFile() {
+ try (BufferedWriter writer = new BufferedWriter(new FileWriter(schemaFile))) {
+ writer.write("table_name=root.db1");
+ writer.newLine();
+ writer.write("time_precision=ms");
+ writer.newLine();
+ writer.write("has_header=true");
+ writer.newLine();
+ writer.write("separator=,");
+ writer.newLine();
+ writer.write("null_format=\\N");
+ writer.newLine();
+ writer.newLine();
+ writer.write("time_column=time");
+ writer.newLine();
+ writer.write("csv_columns");
+ writer.newLine();
+ writer.write("time INT64,");
+ writer.newLine();
+ writer.write("tmp2 FLOAT,");
+ writer.newLine();
+ writer.write("tmp3 FLOAT,");
+ writer.newLine();
+ writer.write("SKIP,");
+ writer.newLine();
+ writer.write("tmp5 FLOAT");
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ public void genCsvFile(int rows) {
+
+ try (BufferedWriter writer = new BufferedWriter(new FileWriter(csvFile))) {
+ writer.write("time,tmp2,tmp3,tmp4,tmp5");
+ writer.newLine();
+ Random random = new Random();
+ long timestamp = System.currentTimeMillis();
+
+ for (int i = 0; i < rows; i++) {
+ timestamp = timestamp + i;
+ float tmp2 = random.nextFloat();
+ float tmp3 = random.nextFloat();
+ float tmp4 = random.nextFloat();
+ float tmp5 = random.nextFloat();
+ tmpResult2[i] = tmp2;
+ tmpResult3[i] = tmp3;
+ tmpResult5[i] = tmp5;
+ writer.write(timestamp + "," + tmp2 + "," + tmp3 + "," + tmp4 + "," + tmp5);
+ writer.newLine();
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ FileUtils.deleteDirectory(new File(testDir));
+ }
+
+ @Test
+ public void testCsvToTsfile() throws Exception {
+ String scFilePath = new File(schemaFile).getAbsolutePath();
+ String csvFilePath = new File(csvFile).getAbsolutePath();
+ String targetPath = new File(testDir).getAbsolutePath();
+ String dataTsfilePath = new File(targetPath + File.separator + "data.tsfile").getAbsolutePath();
+ String[] args = new String[] {"-s" + csvFilePath, "-schema" + scFilePath, "-t" + targetPath};
+ TsFileTool.main(args);
+ List<String> columns = new ArrayList<>();
+ columns.add("tmp2");
+ columns.add("tmp3");
+ columns.add("tmp5");
+ try (TsFileSequenceReader sequenceReader = new TsFileSequenceReader(dataTsfilePath)) {
+ TableQueryExecutor tableQueryExecutor =
+ new TableQueryExecutor(
+ new MetadataQuerierByFileImpl(sequenceReader),
+ new CachedChunkLoaderImpl(sequenceReader),
+ TableQueryExecutor.TableQueryOrdering.DEVICE);
+ final TsBlockReader reader = tableQueryExecutor.query("root.db1", columns, null, null, null);
+ assertTrue(reader.hasNext());
+ int cnt = 0;
+ while (reader.hasNext()) {
+ final TsBlock result = reader.next();
+ float[] floats_tmp2 = result.getColumn(0).getFloats();
+ float[] floats_tmp3 = result.getColumn(1).getFloats();
+ float[] floats_tmp5 = result.getColumn(2).getFloats();
+ for (int i = 0; i < 20; i++) {
+ assertEquals(tmpResult2[i], floats_tmp2[i], 0);
+ assertEquals(tmpResult3[i], floats_tmp3[i], 0);
+ assertEquals(tmpResult5[i], floats_tmp5[i], 0);
+ }
+ cnt += result.getPositionCount();
+ }
+ assertEquals(20, cnt);
+ }
+ }
+}
diff --git a/java/tsfile/pom.xml b/java/tsfile/pom.xml
index 32466d39..8ab97881 100644
--- a/java/tsfile/pom.xml
+++ b/java/tsfile/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.tsfile</groupId>
<artifactId>tsfile-java</artifactId>
- <version>1.0.1-a6fb416-SNAPSHOT</version>
+ <version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>tsfile</artifactId>
<name>TsFile: Java: TsFile</name>
@@ -38,7 +38,7 @@
<dependency>
<groupId>org.apache.tsfile</groupId>
<artifactId>common</artifactId>
- <version>1.0.1-a6fb416-SNAPSHOT</version>
+ <version>1.2.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.github.luben</groupId>
@@ -213,6 +213,14 @@
</execution>
</executions>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>8</source>
+ <target>8</target>
+ </configuration>
+ </plugin>
</plugins>
</build>
<profiles>
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/common/cache/LRUCache.java b/java/tsfile/src/main/java/org/apache/tsfile/common/cache/LRUCache.java
index 9b79825b..a00469a5 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/common/cache/LRUCache.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/common/cache/LRUCache.java
@@ -51,6 +51,10 @@ public abstract class LRUCache<K, T> implements Cache<K, T> {
}
}
+ public boolean containsKey(K key) {
+ return cache.containsKey(key);
+ }
+
@Override
public synchronized void clear() {
cache.clear();
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java b/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java
index 5c0b719a..1cdd59e3 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java
@@ -65,7 +65,9 @@ public class TSFileConfig implements Serializable {
public static final String VERSION_NUMBER_V1 = "000001";
/** version number is changed to use 1 byte to represent since version 3. */
- public static final byte VERSION_NUMBER = 0x03;
+ public static final byte VERSION_NUMBER_V3 = 0x03;
+
+ public static final byte VERSION_NUMBER = 0x04;
/** Bloom filter constrain. */
public static final double MIN_BLOOM_FILTER_ERROR_RATE = 0.01;
@@ -75,6 +77,9 @@ public class TSFileConfig implements Serializable {
/** The primitive array capacity threshold. */
public static final int ARRAY_CAPACITY_THRESHOLD = 1000;
+ // TODO: configurable but unchangeable
+ public static int DEFAULT_SEGMENT_NUM_FOR_TABLE_NAME = 3;
+
/** Memory size threshold for flushing to disk, default value is 128MB. */
private int groupSizeInByte = 128 * 1024 * 1024;
diff --git a/cpp/examples/c_examples/c_examples.h b/java/tsfile/src/main/java/org/apache/tsfile/compatibility/BufferDeserializer.java
similarity index 82%
copy from cpp/examples/c_examples/c_examples.h
copy to java/tsfile/src/main/java/org/apache/tsfile/compatibility/BufferDeserializer.java
index d29090cb..2bda7063 100644
--- a/cpp/examples/c_examples/c_examples.h
+++ b/java/tsfile/src/main/java/org/apache/tsfile/compatibility/BufferDeserializer.java
@@ -17,16 +17,10 @@
* under the License.
*/
+package org.apache.tsfile.compatibility;
-#include "cwrapper/TsFile-cwrapper.h"
+import java.nio.ByteBuffer;
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-ErrorCode write_tsfile();
-ErrorCode read_tsfile();
-
-#ifdef __cplusplus
+public interface BufferDeserializer<T> {
+ T deserialize(ByteBuffer buffer, DeserializeConfig context);
}
-#endif
\ No newline at end of file
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/compatibility/CompatibilityUtils.java b/java/tsfile/src/main/java/org/apache/tsfile/compatibility/CompatibilityUtils.java
new file mode 100644
index 00000000..de01351f
--- /dev/null
+++ b/java/tsfile/src/main/java/org/apache/tsfile/compatibility/CompatibilityUtils.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tsfile.compatibility;
+
+import org.apache.tsfile.file.metadata.MetadataIndexNode;
+import org.apache.tsfile.file.metadata.PlainDeviceID;
+import org.apache.tsfile.file.metadata.TsFileMetadata;
+import org.apache.tsfile.utils.BloomFilter;
+import org.apache.tsfile.utils.ReadWriteForEncodingUtils;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+
+import java.nio.ByteBuffer;
+import java.util.Collections;
+
+public class CompatibilityUtils {
+
+ private CompatibilityUtils() {
+ // util class
+ }
+
+ public static DeserializeConfig v3DeserializeConfig = new DeserializeConfig();
+
+ static {
+ v3DeserializeConfig.tsFileMetadataBufferDeserializer =
+ CompatibilityUtils::deserializeTsFileMetadataFromV3;
+ v3DeserializeConfig.deviceIDBufferDeserializer =
+ ((buffer, context) -> {
+ final PlainDeviceID deviceID = PlainDeviceID.deserialize(buffer);
+ return deviceID.convertToStringArrayDeviceId();
+ });
+ }
+
+ public static TsFileMetadata deserializeTsFileMetadataFromV3(
+ ByteBuffer buffer, DeserializeConfig context) {
+ TsFileMetadata fileMetaData = new TsFileMetadata();
+
+ // metadataIndex
+ MetadataIndexNode metadataIndexNode =
+ context.deviceMetadataIndexNodeBufferDeserializer.deserialize(buffer, context);
+ fileMetaData.setTableMetadataIndexNodeMap(Collections.singletonMap("", metadataIndexNode));
+
+ // metaOffset
+ long metaOffset = ReadWriteIOUtils.readLong(buffer);
+ fileMetaData.setMetaOffset(metaOffset);
+
+ // read bloom filter
+ if (buffer.hasRemaining()) {
+ byte[] bytes = ReadWriteIOUtils.readByteBufferWithSelfDescriptionLength(buffer);
+ int filterSize = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer);
+ int hashFunctionSize = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer);
+ fileMetaData.setBloomFilter(
+ BloomFilter.buildBloomFilter(bytes, filterSize, hashFunctionSize));
+ }
+
+ return fileMetaData;
+ }
+}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/compatibility/DeserializeConfig.java b/java/tsfile/src/main/java/org/apache/tsfile/compatibility/DeserializeConfig.java
new file mode 100644
index 00000000..b1c4f18f
--- /dev/null
+++ b/java/tsfile/src/main/java/org/apache/tsfile/compatibility/DeserializeConfig.java
@@ -0,0 +1,103 @@
+package org.apache.tsfile.compatibility;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.tsfile.file.IMetadataIndexEntry;
+import org.apache.tsfile.file.metadata.DeviceMetadataIndexEntry;
+import org.apache.tsfile.file.metadata.IDeviceID;
+import org.apache.tsfile.file.metadata.MeasurementMetadataIndexEntry;
+import org.apache.tsfile.file.metadata.MetadataIndexNode;
+import org.apache.tsfile.file.metadata.StringArrayDeviceID;
+import org.apache.tsfile.file.metadata.TableSchema;
+import org.apache.tsfile.file.metadata.TsFileMetadata;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+
+public class DeserializeConfig {
+ public BufferDeserializer<TsFileMetadata> tsFileMetadataBufferDeserializer =
+ TsFileMetadata::deserializeFrom;
+
+ public BufferDeserializer<MetadataIndexNode> deviceMetadataIndexNodeBufferDeserializer =
+ (buffer, context) -> MetadataIndexNode.deserializeFrom(buffer, true, context);
+ public BufferDeserializer<MetadataIndexNode> measurementMetadataIndexNodeBufferDeserializer =
+ (buffer, context) -> MetadataIndexNode.deserializeFrom(buffer, false, context);
+ public BufferDeserializer<IMetadataIndexEntry> deviceMetadataIndexEntryBufferDeserializer =
+ DeviceMetadataIndexEntry::deserializeFrom;
+ public BufferDeserializer<IMetadataIndexEntry> measurementMetadataIndexEntryBufferDeserializer =
+ ((buffer, context) -> MeasurementMetadataIndexEntry.deserializeFrom(buffer));
+
+ public BufferDeserializer<TableSchema> tableSchemaBufferDeserializer = TableSchema::deserialize;
+ public BufferDeserializer<MeasurementSchema> measurementSchemaBufferDeserializer =
+ ((buffer, context) -> MeasurementSchema.deserializeFrom(buffer));
+
+ public BufferDeserializer<IDeviceID> deviceIDBufferDeserializer =
+ ((buffer, context) -> StringArrayDeviceID.deserialize(buffer));
+
+ // stream deserializers
+ public StreamDeserializer<MetadataIndexNode> deviceMetadataIndexNodeStreamDeserializer =
+ (stream, context) -> MetadataIndexNode.deserializeFrom(stream, true, context);
+ public StreamDeserializer<MetadataIndexNode> measurementMetadataIndexNodeStreamDeserializer =
+ (stream, context) -> MetadataIndexNode.deserializeFrom(stream, false, context);
+ public StreamDeserializer<IMetadataIndexEntry> deviceMetadataIndexEntryStreamDeserializer =
+ DeviceMetadataIndexEntry::deserializeFrom;
+ public StreamDeserializer<IMetadataIndexEntry> measurementMetadataIndexEntryStreamDeserializer =
+ ((stream, context) -> MeasurementMetadataIndexEntry.deserializeFrom(stream));
+
+ public StreamDeserializer<IDeviceID> deviceIDStreamDeserializer =
+ ((stream, context) -> StringArrayDeviceID.deserialize(stream));
+
+ public MetadataIndexNode deserializeMetadataIndexNode(ByteBuffer buffer, boolean isDeviceLevel) {
+ if (isDeviceLevel) {
+ return deviceMetadataIndexNodeBufferDeserializer.deserialize(buffer, this);
+ } else {
+ return measurementMetadataIndexNodeBufferDeserializer.deserialize(buffer, this);
+ }
+ }
+
+ public IMetadataIndexEntry deserializeMetadataIndexEntry(
+ ByteBuffer buffer, boolean isDeviceLevel) {
+ if (isDeviceLevel) {
+ return deviceMetadataIndexEntryBufferDeserializer.deserialize(buffer, this);
+ } else {
+ return measurementMetadataIndexEntryBufferDeserializer.deserialize(buffer, this);
+ }
+ }
+
+ public MetadataIndexNode deserializeMetadataIndexNode(InputStream stream, boolean isDeviceLevel)
+ throws IOException {
+ if (isDeviceLevel) {
+ return deviceMetadataIndexNodeStreamDeserializer.deserialize(stream, this);
+ } else {
+ return measurementMetadataIndexNodeStreamDeserializer.deserialize(stream, this);
+ }
+ }
+
+ public IMetadataIndexEntry deserializeMetadataIndexEntry(
+ InputStream stream, boolean isDeviceLevel) throws IOException {
+ if (isDeviceLevel) {
+ return deviceMetadataIndexEntryStreamDeserializer.deserialize(stream, this);
+ } else {
+ return measurementMetadataIndexEntryStreamDeserializer.deserialize(stream, this);
+ }
+ }
+}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeEnum.java b/java/tsfile/src/main/java/org/apache/tsfile/compatibility/StreamDeserializer.java
similarity index 78%
copy from java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeEnum.java
copy to java/tsfile/src/main/java/org/apache/tsfile/compatibility/StreamDeserializer.java
index 5a4489de..b4870ff8 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeEnum.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/compatibility/StreamDeserializer.java
@@ -17,18 +17,11 @@
* under the License.
*/
-package org.apache.tsfile.read.common.type;
+package org.apache.tsfile.compatibility;
-public enum TypeEnum {
- INT32,
+import java.io.IOException;
+import java.io.InputStream;
- INT64,
-
- FLOAT,
-
- DOUBLE,
-
- BOOLEAN,
-
- BINARY
+public interface StreamDeserializer<T> {
+ T deserialize(InputStream inputStream, DeserializeConfig context) throws IOException;
}
diff --git a/cpp/examples/c_examples/c_examples.h b/java/tsfile/src/main/java/org/apache/tsfile/exception/IllegalDeviceIDException.java
similarity index 82%
copy from cpp/examples/c_examples/c_examples.h
copy to java/tsfile/src/main/java/org/apache/tsfile/exception/IllegalDeviceIDException.java
index d29090cb..73b82be5 100644
--- a/cpp/examples/c_examples/c_examples.h
+++ b/java/tsfile/src/main/java/org/apache/tsfile/exception/IllegalDeviceIDException.java
@@ -17,16 +17,11 @@
* under the License.
*/
+package org.apache.tsfile.exception;
-#include "cwrapper/TsFile-cwrapper.h"
+public class IllegalDeviceIDException extends RuntimeException {
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-ErrorCode write_tsfile();
-ErrorCode read_tsfile();
-
-#ifdef __cplusplus
+ public IllegalDeviceIDException(String message) {
+ super(message);
+ }
}
-#endif
\ No newline at end of file
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/exception/TsFileRuntimeException.java b/java/tsfile/src/main/java/org/apache/tsfile/exception/TsFileRuntimeException.java
index 9bbd16e8..fe9fa934 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/exception/TsFileRuntimeException.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/exception/TsFileRuntimeException.java
@@ -30,4 +30,8 @@ public class TsFileRuntimeException extends RuntimeException {
public TsFileRuntimeException(String message) {
super(message);
}
+
+ public TsFileRuntimeException(Throwable cause) {
+ super(cause);
+ }
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/IMetadataIndexEntry.java b/java/tsfile/src/main/java/org/apache/tsfile/exception/read/FileVersionTooOldException.java
similarity index 70%
copy from java/tsfile/src/main/java/org/apache/tsfile/file/IMetadataIndexEntry.java
copy to java/tsfile/src/main/java/org/apache/tsfile/exception/read/FileVersionTooOldException.java
index cc1cea3b..2176f7ff 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/file/IMetadataIndexEntry.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/exception/read/FileVersionTooOldException.java
@@ -17,20 +17,16 @@
* under the License.
*/
-package org.apache.tsfile.file;
+package org.apache.tsfile.exception.read;
import java.io.IOException;
-import java.io.OutputStream;
-public interface IMetadataIndexEntry {
+public class FileVersionTooOldException extends IOException {
- long getOffset();
-
- void setOffset(long offset);
-
- int serializeTo(OutputStream outputStream) throws IOException;
-
- Comparable getCompareKey();
-
- boolean isDeviceLevel();
+ public FileVersionTooOldException(byte currentVersion, byte minimumVersion) {
+ super(
+ String.format(
+ "The current version %d is too old, please at least upgrade to %d",
+ currentVersion, minimumVersion));
+ }
}
diff --git a/cpp/examples/c_examples/c_examples.h b/java/tsfile/src/main/java/org/apache/tsfile/exception/read/NoColumnException.java
similarity index 79%
copy from cpp/examples/c_examples/c_examples.h
copy to java/tsfile/src/main/java/org/apache/tsfile/exception/read/NoColumnException.java
index d29090cb..ee227092 100644
--- a/cpp/examples/c_examples/c_examples.h
+++ b/java/tsfile/src/main/java/org/apache/tsfile/exception/read/NoColumnException.java
@@ -17,16 +17,11 @@
* under the License.
*/
+package org.apache.tsfile.exception.read;
-#include "cwrapper/TsFile-cwrapper.h"
+public class NoColumnException extends ReadProcessException {
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-ErrorCode write_tsfile();
-ErrorCode read_tsfile();
-
-#ifdef __cplusplus
+ public NoColumnException(String columnName) {
+ super(String.format("No column: %s", columnName));
+ }
}
-#endif
\ No newline at end of file
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/exception/write/NoMeasurementException.java b/java/tsfile/src/main/java/org/apache/tsfile/exception/read/ReadProcessException.java
similarity index 71%
copy from java/tsfile/src/main/java/org/apache/tsfile/exception/write/NoMeasurementException.java
copy to java/tsfile/src/main/java/org/apache/tsfile/exception/read/ReadProcessException.java
index 5e5745e2..7bd7c77e 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/exception/write/NoMeasurementException.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/exception/read/ReadProcessException.java
@@ -17,14 +17,19 @@
* under the License.
*/
-package org.apache.tsfile.exception.write;
+package org.apache.tsfile.exception.read;
-/** This exception means it can not find the measurement while writing a TSRecord. */
-public class NoMeasurementException extends WriteProcessException {
+public class ReadProcessException extends Exception {
- private static final long serialVersionUID = -5599767368831572747L;
+ public ReadProcessException(String message) {
+ super(message);
+ }
+
+ public ReadProcessException(String message, Throwable cause) {
+ super(message, cause);
+ }
- public NoMeasurementException(String msg) {
- super(msg);
+ public ReadProcessException(Throwable cause) {
+ super(cause);
}
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeEnum.java b/java/tsfile/src/main/java/org/apache/tsfile/exception/read/UnsupportedOrderingException.java
similarity index 77%
copy from java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeEnum.java
copy to java/tsfile/src/main/java/org/apache/tsfile/exception/read/UnsupportedOrderingException.java
index 5a4489de..6b7120c7 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeEnum.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/exception/read/UnsupportedOrderingException.java
@@ -17,18 +17,11 @@
* under the License.
*/
-package org.apache.tsfile.read.common.type;
+package org.apache.tsfile.exception.read;
-public enum TypeEnum {
- INT32,
+public class UnsupportedOrderingException extends ReadProcessException {
- INT64,
-
- FLOAT,
-
- DOUBLE,
-
- BOOLEAN,
-
- BINARY
+ public UnsupportedOrderingException(String ordering) {
+ super(String.format("Unsupported ordering: %s", ordering));
+ }
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/exception/write/NoMeasurementException.java b/java/tsfile/src/main/java/org/apache/tsfile/exception/write/ConflictDataTypeException.java
similarity index 73%
copy from java/tsfile/src/main/java/org/apache/tsfile/exception/write/NoMeasurementException.java
copy to java/tsfile/src/main/java/org/apache/tsfile/exception/write/ConflictDataTypeException.java
index 5e5745e2..da0cee54 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/exception/write/NoMeasurementException.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/exception/write/ConflictDataTypeException.java
@@ -19,12 +19,12 @@
package org.apache.tsfile.exception.write;
-/** This exception means it can not find the measurement while writing a TSRecord. */
-public class NoMeasurementException extends WriteProcessException {
+import org.apache.tsfile.enums.TSDataType;
- private static final long serialVersionUID = -5599767368831572747L;
+public class ConflictDataTypeException extends WriteProcessException {
- public NoMeasurementException(String msg) {
- super(msg);
+ public ConflictDataTypeException(TSDataType writing, TSDataType registered) {
+ super(
+ String.format("Conflict data type: %s (writing) and %s (registered)", writing, registered));
}
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/exception/write/NoMeasurementException.java b/java/tsfile/src/main/java/org/apache/tsfile/exception/write/NoDeviceException.java
similarity index 85%
copy from java/tsfile/src/main/java/org/apache/tsfile/exception/write/NoMeasurementException.java
copy to java/tsfile/src/main/java/org/apache/tsfile/exception/write/NoDeviceException.java
index 5e5745e2..d72cb829 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/exception/write/NoMeasurementException.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/exception/write/NoDeviceException.java
@@ -20,11 +20,11 @@
package org.apache.tsfile.exception.write;
/** This exception means it can not find the measurement while writing a TSRecord. */
-public class NoMeasurementException extends WriteProcessException {
+public class NoDeviceException extends WriteProcessException {
private static final long serialVersionUID = -5599767368831572747L;
- public NoMeasurementException(String msg) {
- super(msg);
+ public NoDeviceException(String columnName) {
+ super(String.format("No device for %s", columnName));
}
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/exception/write/NoMeasurementException.java b/java/tsfile/src/main/java/org/apache/tsfile/exception/write/NoMeasurementException.java
index 5e5745e2..b7d3f3a9 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/exception/write/NoMeasurementException.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/exception/write/NoMeasurementException.java
@@ -24,7 +24,7 @@ public class NoMeasurementException extends WriteProcessException {
private static final long serialVersionUID = -5599767368831572747L;
- public NoMeasurementException(String msg) {
- super(msg);
+ public NoMeasurementException(String columnName) {
+ super(String.format("No measurement for %s", columnName));
}
}
diff --git a/cpp/examples/c_examples/c_examples.h b/java/tsfile/src/main/java/org/apache/tsfile/exception/write/NoTableException.java
similarity index 79%
copy from cpp/examples/c_examples/c_examples.h
copy to java/tsfile/src/main/java/org/apache/tsfile/exception/write/NoTableException.java
index d29090cb..5098916d 100644
--- a/cpp/examples/c_examples/c_examples.h
+++ b/java/tsfile/src/main/java/org/apache/tsfile/exception/write/NoTableException.java
@@ -17,16 +17,11 @@
* under the License.
*/
+package org.apache.tsfile.exception.write;
-#include "cwrapper/TsFile-cwrapper.h"
+public class NoTableException extends WriteProcessException {
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-ErrorCode write_tsfile();
-ErrorCode read_tsfile();
-
-#ifdef __cplusplus
+ public NoTableException(String tableName) {
+ super(String.format("Table %s not found", tableName));
+ }
}
-#endif
\ No newline at end of file
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/IMetadataIndexEntry.java b/java/tsfile/src/main/java/org/apache/tsfile/file/IMetadataIndexEntry.java
index cc1cea3b..4a6f08c2 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/file/IMetadataIndexEntry.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/file/IMetadataIndexEntry.java
@@ -33,4 +33,6 @@ public interface IMetadataIndexEntry {
Comparable getCompareKey();
boolean isDeviceLevel();
+
+ int serializedSize();
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/header/ChunkGroupHeader.java b/java/tsfile/src/main/java/org/apache/tsfile/file/header/ChunkGroupHeader.java
index 8e327af6..7a424f89 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/file/header/ChunkGroupHeader.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/file/header/ChunkGroupHeader.java
@@ -19,10 +19,8 @@
package org.apache.tsfile.file.header;
-import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.file.MetaMarker;
import org.apache.tsfile.file.metadata.IDeviceID;
-import org.apache.tsfile.file.metadata.PlainDeviceID;
import org.apache.tsfile.read.reader.TsFileInput;
import org.apache.tsfile.utils.ReadWriteForEncodingUtils;
import org.apache.tsfile.utils.ReadWriteIOUtils;
@@ -56,8 +54,7 @@ public class ChunkGroupHeader {
private int getSerializedSize(IDeviceID deviceID) {
// TODO: add an interface in IDeviceID
- int length =
- ((PlainDeviceID) deviceID).toStringID().getBytes(TSFileConfig.STRING_CHARSET).length;
+ int length = deviceID.serializedSize();
return Byte.BYTES + ReadWriteForEncodingUtils.varIntSize(length) + length;
}
@@ -77,11 +74,9 @@ public class ChunkGroupHeader {
}
// TODO: add an interface in IDeviceID
- String deviceID = ReadWriteIOUtils.readVarIntString(inputStream);
- if (deviceID == null || deviceID.isEmpty()) {
- throw new IOException("DeviceId is empty");
- }
- return new ChunkGroupHeader(new PlainDeviceID(deviceID));
+ final IDeviceID deviceID =
+ IDeviceID.Deserializer.DEFAULT_DESERIALIZER.deserializeFrom(inputStream);
+ return new ChunkGroupHeader(deviceID);
}
/**
@@ -96,9 +91,11 @@ public class ChunkGroupHeader {
if (!markerRead) {
offsetVar++;
}
- // TODO: add an interface in IDeviceID
- String deviceID = input.readVarIntString(offsetVar);
- return new ChunkGroupHeader(new PlainDeviceID(deviceID));
+ input.position(offsetVar);
+ final InputStream inputStream = input.wrapAsInputStream();
+ final IDeviceID deviceID =
+ IDeviceID.Deserializer.DEFAULT_DESERIALIZER.deserializeFrom(inputStream);
+ return new ChunkGroupHeader(deviceID);
}
public IDeviceID getDeviceID() {
@@ -115,7 +112,7 @@ public class ChunkGroupHeader {
public int serializeTo(OutputStream outputStream) throws IOException {
int length = 0;
length += ReadWriteIOUtils.write(MARKER, outputStream);
- length += ReadWriteIOUtils.writeVar(((PlainDeviceID) deviceID).toStringID(), outputStream);
+ length += deviceID.serialize(outputStream);
return length;
}
@@ -123,7 +120,7 @@ public class ChunkGroupHeader {
public String toString() {
return "ChunkGroupHeader{"
+ "deviceID='"
- + ((PlainDeviceID) deviceID).toStringID()
+ + deviceID
+ '\''
+ ", serializedSize="
+ serializedSize
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/header/PageHeader.java b/java/tsfile/src/main/java/org/apache/tsfile/file/header/PageHeader.java
index 70387419..78d80f32 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/file/header/PageHeader.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/file/header/PageHeader.java
@@ -107,6 +107,12 @@ public class PageHeader implements IMetadata {
return statistics.getCount();
}
+ public int getSerializedSize() {
+ return ReadWriteForEncodingUtils.varIntSize(uncompressedSize)
+ + ReadWriteForEncodingUtils.varIntSize(compressedSize)
+ + (statistics != null ? statistics.getSerializedSize() : 0);
+ }
+
@Override
public Statistics<? extends Serializable> getStatistics() {
return statistics;
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/ChunkMetadata.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/ChunkMetadata.java
index e491d7bb..5ce097fe 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/ChunkMetadata.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/ChunkMetadata.java
@@ -20,11 +20,14 @@
package org.apache.tsfile.file.metadata;
import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.enums.CompressionType;
+import org.apache.tsfile.file.metadata.enums.TSEncoding;
import org.apache.tsfile.file.metadata.statistics.Statistics;
import org.apache.tsfile.read.common.TimeRange;
import org.apache.tsfile.read.controller.IChunkLoader;
import org.apache.tsfile.utils.RamUsageEstimator;
import org.apache.tsfile.utils.ReadWriteIOUtils;
+import org.apache.tsfile.write.schema.MeasurementSchema;
import java.io.IOException;
import java.io.InputStream;
@@ -55,6 +58,10 @@ public class ChunkMetadata implements IChunkMetadata {
private TSDataType tsDataType;
+ // the following two are not serialized and only used during write
+ private TSEncoding encoding;
+ private CompressionType compressionType;
+
/**
* version is used to define the order of operations(insertion, deletion, update). version is set
* according to its belonging ChunkGroup only when being queried, so it is not persisted.
@@ -93,12 +100,16 @@ public class ChunkMetadata implements IChunkMetadata {
public ChunkMetadata(
String measurementUid,
TSDataType tsDataType,
+ TSEncoding encoding,
+ CompressionType compressionType,
long fileOffset,
Statistics<? extends Serializable> statistics) {
this.measurementUid = measurementUid;
this.tsDataType = tsDataType;
this.offsetOfChunkHeader = fileOffset;
this.statistics = statistics;
+ this.encoding = encoding;
+ this.compressionType = compressionType;
}
// won't clone deleteIntervalList & modified
@@ -111,6 +122,8 @@ public class ChunkMetadata implements IChunkMetadata {
this.isSeq = other.isSeq;
this.isClosed = other.isClosed;
this.mask = other.mask;
+ this.encoding = other.encoding;
+ this.compressionType = other.compressionType;
}
@Override
@@ -171,6 +184,14 @@ public class ChunkMetadata implements IChunkMetadata {
return byteLen;
}
+ public int serializedSize(boolean includeStatistics) {
+ int cnt = Long.BYTES; // offsetOfChunkHeader
+ if (includeStatistics) {
+ cnt += statistics.getSerializedSize();
+ }
+ return cnt;
+ }
+
/**
* deserialize from ByteBuffer.
*
@@ -384,4 +405,9 @@ public class ChunkMetadata implements IChunkMetadata {
public boolean hasNullValue(int measurementIndex) {
return false;
}
+
+ // TODO: replaces fields in this class with MeasurementSchema
+ public MeasurementSchema toMeasurementSchema() {
+ return new MeasurementSchema(measurementUid, tsDataType, encoding, compressionType);
+ }
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/DeviceMetadataIndexEntry.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/DeviceMetadataIndexEntry.java
index 9993cb7a..3c7c146e 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/DeviceMetadataIndexEntry.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/DeviceMetadataIndexEntry.java
@@ -19,6 +19,7 @@
package org.apache.tsfile.file.metadata;
+import org.apache.tsfile.compatibility.DeserializeConfig;
import org.apache.tsfile.file.IMetadataIndexEntry;
import org.apache.tsfile.utils.ReadWriteIOUtils;
@@ -28,6 +29,7 @@ import java.io.OutputStream;
import java.nio.ByteBuffer;
public class DeviceMetadataIndexEntry implements IMetadataIndexEntry {
+
private IDeviceID deviceID;
private long offset;
@@ -72,19 +74,25 @@ public class DeviceMetadataIndexEntry implements IMetadataIndexEntry {
return true;
}
- public static DeviceMetadataIndexEntry deserializeFrom(ByteBuffer buffer) {
- IDeviceID device = IDeviceID.deserializeFrom(buffer);
+ public static DeviceMetadataIndexEntry deserializeFrom(
+ ByteBuffer buffer, DeserializeConfig context) {
+ IDeviceID device = context.deviceIDBufferDeserializer.deserialize(buffer, context);
long offset = ReadWriteIOUtils.readLong(buffer);
return new DeviceMetadataIndexEntry(device, offset);
}
- public static DeviceMetadataIndexEntry deserializeFrom(InputStream inputStream)
- throws IOException {
- IDeviceID device = IDeviceID.deserializeFrom(inputStream);
+ public static DeviceMetadataIndexEntry deserializeFrom(
+ InputStream inputStream, DeserializeConfig config) throws IOException {
+ IDeviceID device = config.deviceIDStreamDeserializer.deserialize(inputStream, config);
long offset = ReadWriteIOUtils.readLong(inputStream);
return new DeviceMetadataIndexEntry(device, offset);
}
+ @Override
+ public int serializedSize() {
+ return deviceID.serializedSize() + Long.BYTES; // offset
+ }
+
@Override
public String toString() {
return "<" + deviceID + "," + offset + ">";
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/IDeviceID.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/IDeviceID.java
index 04d73c19..87e04cc0 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/IDeviceID.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/IDeviceID.java
@@ -19,16 +19,23 @@
package org.apache.tsfile.file.metadata;
+import org.apache.tsfile.common.constant.TsFileConstant;
import org.apache.tsfile.utils.Accountable;
-import org.apache.tsfile.utils.ReadWriteIOUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.io.Serializable;
import java.nio.ByteBuffer;
/** Device id interface. */
-public interface IDeviceID extends Comparable<IDeviceID>, Accountable {
+public interface IDeviceID extends Comparable<IDeviceID>, Accountable, Serializable {
+
+ Logger LOGGER = LoggerFactory.getLogger(IDeviceID.class);
int serialize(ByteBuffer byteBuffer);
@@ -38,11 +45,126 @@ public interface IDeviceID extends Comparable<IDeviceID>, Accountable {
boolean isEmpty();
- static IDeviceID deserializeFrom(ByteBuffer byteBuffer) {
- return new PlainDeviceID(ReadWriteIOUtils.readVarIntString(byteBuffer));
+ /**
+ * @return the table name associated with the device. For a path-DeviceId, like "root.a.b.c.d", it
+ * is converted according to a fixed rule, like assuming the first three levels ("root.a.b")
+ * as the table name; for a tuple-deviceId, like "(table1, beijing, turbine)", it is the first
+ * element in the deviceId, namely "table1".
+ */
+ String getTableName();
+
+ /**
+ * @return how many segments this DeviceId consists of. For a path-DeviceId, like "root.a.b.c.d",
+ * it is 5; fot a tuple-DeviceId, like "(table1, beijing, turbine)", it is 3.
+ */
+ int segmentNum();
+
+ /**
+ * @param i the sequence number of the segment that should be returned.
+ * @return i-th segment in this DeviceId.
+ * @throws ArrayIndexOutOfBoundsException if i >= segmentNum().
+ */
+ Object segment(int i);
+
+ default int serializedSize() {
+ LOGGER.debug(
+ "Using default inefficient implementation of serialized size by {}", this.getClass());
+ try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
+ serialize(baos);
+ return baos.size();
+ } catch (IOException e) {
+ LOGGER.error("Failed to serialize device ID: {}", this, e);
+ return -1;
+ }
+ }
+
+ default boolean startWith(String prefix) {
+ return startWith(prefix, false);
+ }
+
+ /**
+ * @param prefix prefix to be matched, with "." as segment separator
+ * @param matchEntireSegment if true, the prefix should match entire segments. E.g., "root.a.b"
+ * matches ("root", "a", "bb") if matchEntireSegment is false; but it mismatches when
+ * matchEntireSegment is true.
+ * @return true if the prefix can be matched, false other with
+ */
+ default boolean startWith(String prefix, boolean matchEntireSegment) {
+ int currSegment = 0;
+ int matchedPos = 0;
+ while (currSegment < segmentNum()) {
+ String segmentString = segment(currSegment).toString();
+ String remainingPrefix = prefix.substring(matchedPos);
+ if (segmentString.startsWith(remainingPrefix)) {
+ // ("root.a.b","c","d") matches "root.a", "root.a.b", "root.a.b.c", "root.a.b.c.d"
+ if (matchEntireSegment) {
+ // ("root.a.b","c","d") matches "root.a.b", "root.a.b.c", "root.a.b.c.d"
+ return segmentString.equals(remainingPrefix);
+ }
+ return true;
+ }
+ if (!remainingPrefix.startsWith(segmentString)) {
+ // ("root.a.b","c","d") mismatches "root.b", "root.a.b.d", "root.a.b.c.e"
+ return false;
+ }
+ // the current segment is fully matched
+ matchedPos += segmentString.length();
+ // check path separator
+ if (prefix.charAt(matchedPos) != TsFileConstant.PATH_SEPARATOR_CHAR) {
+ // ("root.a.b","c","d") mismatches "root.a.bb", "root.a.b.cc", "root.a.b.c.dd"
+ return false;
+ }
+ // path separator is matched, move to the next segment
+ matchedPos++;
+ currSegment++;
+ }
+ // ("root.a.b","c","d") mismatches "root.a.b.c.d.e"
+ return false;
+ }
+
+ default Object[] getSegments() {
+ final Object[] segments = new Object[segmentNum()];
+ for (int i = 0; i < segmentNum(); i++) {
+ segments[i] = segment(i);
+ }
+ return segments;
}
- static IDeviceID deserializeFrom(InputStream inputStream) throws IOException {
- return new PlainDeviceID(ReadWriteIOUtils.readVarIntString(inputStream));
+ default boolean matchDatabaseName(String databaseName) {
+ String tableName = getTableName();
+ if (tableName.startsWith(databaseName)
+ // root.aa.bb matches root.aa.bb
+ && (tableName.length() == databaseName.length()
+ ||
+ // root.aa.bb matches root.aa
+ tableName.charAt(databaseName.length()) == TsFileConstant.PATH_SEPARATOR_CHAR)) {
+ return true;
+ }
+ // root.aa mismatches root.a
+ return startWith(databaseName, true);
+ }
+
+ interface Deserializer {
+
+ IDeviceID deserializeFrom(ByteBuffer byteBuffer);
+
+ IDeviceID deserializeFrom(InputStream inputStream) throws IOException;
+
+ Deserializer DEFAULT_DESERIALIZER = StringArrayDeviceID.getDESERIALIZER();
+ }
+
+ interface Factory {
+
+ IDeviceID create(String deviceIdString);
+
+ /**
+ * The first segment is the table name and the rests are id columns.
+ *
+ * @param segments Example: ["table0", "id0", "id1"]
+ * @return a deviceId corresponding to the segments
+ */
+ IDeviceID create(String[] segments);
+
+ Factory DEFAULT_FACTORY = StringArrayDeviceID.getFACTORY();
}
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/LogicalTableSchema.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/LogicalTableSchema.java
new file mode 100644
index 00000000..2db59473
--- /dev/null
+++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/LogicalTableSchema.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tsfile.file.metadata;
+
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.enums.CompressionType;
+import org.apache.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.tsfile.write.record.Tablet.ColumnType;
+import org.apache.tsfile.write.schema.IMeasurementSchema;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * TableSchema for devices with path-based DeviceIds. It generates the Id columns based on the max
+ * level of paths.
+ */
+public class LogicalTableSchema extends TableSchema {
+
+ private int maxLevel;
+
+ public LogicalTableSchema(String tableName) {
+ super(tableName);
+ }
+
+ @Override
+ public void update(ChunkGroupMetadata chunkGroupMetadata) {
+ super.update(chunkGroupMetadata);
+ this.maxLevel = Math.max(this.maxLevel, chunkGroupMetadata.getDevice().segmentNum());
+ }
+
+ private List<IMeasurementSchema> generateIdColumns() {
+ List<IMeasurementSchema> generatedIdColumns = new ArrayList<>();
+ // level 0 is table name, not id column
+ for (int i = 1; i < maxLevel; i++) {
+ generatedIdColumns.add(
+ new MeasurementSchema(
+ "__level" + i, TSDataType.STRING, TSEncoding.PLAIN, CompressionType.UNCOMPRESSED));
+ }
+ return generatedIdColumns;
+ }
+
+ public void finalizeColumnSchema() {
+ if (!updatable) {
+ return;
+ }
+
+ List<IMeasurementSchema> allColumns = new ArrayList<>(generateIdColumns());
+ List<ColumnType> allColumnTypes = ColumnType.nCopy(ColumnType.ID, allColumns.size());
+ allColumns.addAll(columnSchemas);
+ allColumnTypes.addAll(columnTypes);
+ columnSchemas = allColumns;
+ columnTypes = allColumnTypes;
+ updatable = false;
+ }
+
+ @Override
+ public int serialize(OutputStream out) throws IOException {
+ finalizeColumnSchema();
+ return super.serialize(out);
+ }
+}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MeasurementMetadataIndexEntry.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MeasurementMetadataIndexEntry.java
index fbb4b4a3..4a794cfd 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MeasurementMetadataIndexEntry.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MeasurementMetadataIndexEntry.java
@@ -20,6 +20,7 @@
package org.apache.tsfile.file.metadata;
import org.apache.tsfile.file.IMetadataIndexEntry;
+import org.apache.tsfile.utils.ReadWriteForEncodingUtils;
import org.apache.tsfile.utils.ReadWriteIOUtils;
import java.io.IOException;
@@ -86,6 +87,11 @@ public class MeasurementMetadataIndexEntry implements IMetadataIndexEntry {
return new MeasurementMetadataIndexEntry(name, offset);
}
+ @Override
+ public int serializedSize() {
+ return ReadWriteForEncodingUtils.varIntStringSize(name) + Long.BYTES; // offset
+ }
+
@Override
public String toString() {
return "<" + name + "," + offset + ">";
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexConstructor.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexConstructor.java
index 3941ec4f..b1afd679 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexConstructor.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexConstructor.java
@@ -88,6 +88,17 @@ public class MetadataIndexConstructor {
return checkAndBuildLevelIndex(deviceMetadataIndexMap, out);
}
+ public static Map<String, Map<IDeviceID, MetadataIndexNode>> splitDeviceByTable(
+ Map<IDeviceID, MetadataIndexNode> deviceMetadataIndexMap) {
+ Map<String, Map<IDeviceID, MetadataIndexNode>> result = new TreeMap<>();
+ for (Entry<IDeviceID, MetadataIndexNode> entry : deviceMetadataIndexMap.entrySet()) {
+ IDeviceID deviceID = entry.getKey();
+ String tableName = deviceID.getTableName();
+ result.computeIfAbsent(tableName, tName -> new TreeMap<>()).put(deviceID, entry.getValue());
+ }
+ return result;
+ }
+
public static MetadataIndexNode checkAndBuildLevelIndex(
Map<IDeviceID, MetadataIndexNode> deviceMetadataIndexMap, TsFileOutput out)
throws IOException {
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java
index 2034ad4f..693b4ffb 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java
@@ -21,6 +21,7 @@ package org.apache.tsfile.file.metadata;
import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.common.conf.TSFileDescriptor;
+import org.apache.tsfile.compatibility.DeserializeConfig;
import org.apache.tsfile.file.IMetadataIndexEntry;
import org.apache.tsfile.file.metadata.enums.MetadataIndexNodeType;
import org.apache.tsfile.utils.Pair;
@@ -98,15 +99,12 @@ public class MetadataIndexNode {
return byteLen;
}
- public static MetadataIndexNode deserializeFrom(ByteBuffer buffer, boolean isDeviceLevel) {
+ public static MetadataIndexNode deserializeFrom(
+ ByteBuffer buffer, boolean isDeviceLevel, DeserializeConfig context) {
List<IMetadataIndexEntry> children = new ArrayList<>();
int size = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer);
for (int i = 0; i < size; i++) {
- if (isDeviceLevel) {
- children.add(DeviceMetadataIndexEntry.deserializeFrom(buffer));
- } else {
- children.add(MeasurementMetadataIndexEntry.deserializeFrom(buffer));
- }
+ children.add(context.deserializeMetadataIndexEntry(buffer, isDeviceLevel));
}
long offset = ReadWriteIOUtils.readLong(buffer);
MetadataIndexNodeType nodeType =
@@ -114,16 +112,12 @@ public class MetadataIndexNode {
return new MetadataIndexNode(children, offset, nodeType);
}
- public static MetadataIndexNode deserializeFrom(InputStream inputStream, boolean isDeviceLevel)
- throws IOException {
+ public static MetadataIndexNode deserializeFrom(
+ InputStream inputStream, boolean isDeviceLevel, DeserializeConfig config) throws IOException {
List<IMetadataIndexEntry> children = new ArrayList<>();
int size = ReadWriteForEncodingUtils.readUnsignedVarInt(inputStream);
for (int i = 0; i < size; i++) {
- if (isDeviceLevel) {
- children.add(DeviceMetadataIndexEntry.deserializeFrom(inputStream));
- } else {
- children.add(MeasurementMetadataIndexEntry.deserializeFrom(inputStream));
- }
+ children.add(config.deserializeMetadataIndexEntry(inputStream, isDeviceLevel));
}
long offset = ReadWriteIOUtils.readLong(inputStream);
MetadataIndexNodeType nodeType =
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/PlainDeviceID.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/PlainDeviceID.java
index eb922521..f36255fe 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/PlainDeviceID.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/PlainDeviceID.java
@@ -19,23 +19,30 @@
package org.apache.tsfile.file.metadata;
+import org.apache.tsfile.common.conf.TSFileConfig;
+import org.apache.tsfile.common.constant.TsFileConstant;
import org.apache.tsfile.utils.RamUsageEstimator;
import org.apache.tsfile.utils.ReadWriteIOUtils;
import java.io.IOException;
+import java.io.InputStream;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.util.Objects;
import static org.apache.tsfile.utils.RamUsageEstimator.sizeOfCharArray;
+// TODO: rename to PathDeviceID (countering TupleDeviceID or ArrayDeviceID)
/** Using device id path as id. */
public class PlainDeviceID implements IDeviceID {
private static final long INSTANCE_SIZE =
RamUsageEstimator.shallowSizeOfInstance(PlainDeviceID.class)
+ + RamUsageEstimator.shallowSizeOfInstance(String.class)
+ RamUsageEstimator.shallowSizeOfInstance(String.class);
private final String deviceID;
+ private String tableName;
+ private String[] segments;
public PlainDeviceID(String deviceID) {
this.deviceID = deviceID;
@@ -59,16 +66,16 @@ public class PlainDeviceID implements IDeviceID {
}
public String toString() {
- return "PlainDeviceID{" + "deviceID='" + deviceID + '\'' + '}';
+ return deviceID;
}
public String toStringID() {
- return deviceID;
+ return toString();
}
@Override
public int serialize(ByteBuffer byteBuffer) {
- return ReadWriteIOUtils.write(deviceID, byteBuffer);
+ return ReadWriteIOUtils.writeVar(deviceID, byteBuffer);
}
@Override
@@ -88,11 +95,27 @@ public class PlainDeviceID implements IDeviceID {
@Override
public long ramBytesUsed() {
- return INSTANCE_SIZE + sizeOfCharArray(deviceID.length());
+ long size = INSTANCE_SIZE;
+ size += sizeOfCharArray(deviceID.length());
+ if (tableName != null) {
+ size += sizeOfCharArray(tableName.length());
+ }
+ if (segments != null) {
+ size += RamUsageEstimator.sizeOf(segments);
+ }
+ return size;
}
public static PlainDeviceID deserialize(ByteBuffer byteBuffer) {
- return new PlainDeviceID(ReadWriteIOUtils.readString(byteBuffer));
+ return new PlainDeviceID(ReadWriteIOUtils.readVarIntString(byteBuffer));
+ }
+
+ public static PlainDeviceID deserialize(InputStream inputStream) throws IOException {
+ return new PlainDeviceID(ReadWriteIOUtils.readVarIntString(inputStream));
+ }
+
+ public StringArrayDeviceID convertToStringArrayDeviceId() {
+ return new StringArrayDeviceID(deviceID);
}
@Override
@@ -102,4 +125,64 @@ public class PlainDeviceID implements IDeviceID {
}
return deviceID.compareTo(((PlainDeviceID) other).deviceID);
}
+
+ @Override
+ public String getTableName() {
+ if (tableName != null) {
+ return tableName;
+ }
+
+ int lastSeparatorPos = -1;
+ int separatorNum = 0;
+
+ for (int i = 0; i < deviceID.length(); i++) {
+ if (deviceID.charAt(i) == TsFileConstant.PATH_SEPARATOR_CHAR) {
+ lastSeparatorPos = i;
+ separatorNum++;
+ if (separatorNum == TSFileConfig.DEFAULT_SEGMENT_NUM_FOR_TABLE_NAME) {
+ break;
+ }
+ }
+ }
+ if (lastSeparatorPos == -1) {
+ // not find even one separator, probably during a test, use the deviceId as the tableName
+ tableName = deviceID;
+ } else {
+ // use the first DEFAULT_SEGMENT_NUM_FOR_TABLE_NAME segments or all segments but the last
+ // one as the table name
+ tableName = deviceID.substring(0, lastSeparatorPos);
+ }
+
+ return tableName;
+ }
+
+ @Override
+ public int segmentNum() {
+ if (segments != null) {
+ return segments.length;
+ }
+ segments = deviceID.split(TsFileConstant.PATH_SEPARATER_NO_REGEX);
+ return segments.length;
+ }
+
+ @Override
+ public String segment(int i) {
+ if (i >= segmentNum()) {
+ throw new ArrayIndexOutOfBoundsException(i);
+ }
+ return segments[i];
+ }
+
+ public static class Factory implements IDeviceID.Factory {
+
+ @Override
+ public IDeviceID create(String deviceIdString) {
+ return new PlainDeviceID(deviceIdString);
+ }
+
+ @Override
+ public IDeviceID create(String[] segments) {
+ return new PlainDeviceID(String.join(TsFileConstant.PATH_SEPARATOR, segments));
+ }
+ }
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/StringArrayDeviceID.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/StringArrayDeviceID.java
new file mode 100644
index 00000000..020b5489
--- /dev/null
+++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/StringArrayDeviceID.java
@@ -0,0 +1,304 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tsfile.file.metadata;
+
+import org.apache.tsfile.common.conf.TSFileConfig;
+import org.apache.tsfile.exception.IllegalDeviceIDException;
+import org.apache.tsfile.exception.TsFileRuntimeException;
+import org.apache.tsfile.read.common.parser.PathNodesGenerator;
+import org.apache.tsfile.utils.RamUsageEstimator;
+import org.apache.tsfile.utils.ReadWriteForEncodingUtils;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+import org.apache.tsfile.utils.WriteUtils;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+
+import static org.apache.tsfile.common.constant.TsFileConstant.PATH_SEPARATOR;
+
+public class StringArrayDeviceID implements IDeviceID {
+
+ private int serializedSize = -1;
+
+ private static final Deserializer DESERIALIZER =
+ new Deserializer() {
+ @Override
+ public IDeviceID deserializeFrom(ByteBuffer byteBuffer) {
+ return deserialize(byteBuffer);
+ }
+
+ @Override
+ public IDeviceID deserializeFrom(InputStream inputStream) throws IOException {
+ return deserialize(inputStream);
+ }
+ };
+
+ private static final Factory FACTORY =
+ new Factory() {
+ @Override
+ public IDeviceID create(String deviceIdString) {
+ return new StringArrayDeviceID(deviceIdString);
+ }
+
+ @Override
+ public IDeviceID create(String[] segments) {
+ return new StringArrayDeviceID(segments);
+ }
+ };
+
+ private static final long INSTANCE_SIZE =
+ RamUsageEstimator.shallowSizeOfInstance(StringArrayDeviceID.class);
+
+ // TODO: change to Object[] and rename to just ArrayDeviceID
+ // or we can just use a tuple like Relational DB.
+ private final String[] segments;
+
+ public StringArrayDeviceID(String... segments) {
+ this.segments = formalize(segments);
+ }
+
+ public StringArrayDeviceID(String deviceIdString) {
+ this.segments = splitDeviceIdString(deviceIdString);
+ }
+
+ private String[] formalize(String[] segments) {
+ // remove tailing nulls
+ int i = segments.length - 1;
+ for (; i >= 0; i--) {
+ if (segments[i] != null) {
+ break;
+ }
+ }
+ if (i < 0) {
+ throw new IllegalDeviceIDException("All segments are null");
+ }
+ if (i != segments.length - 1) {
+ segments = Arrays.copyOf(segments, i + 1);
+ }
+ return segments;
+ }
+
+ @SuppressWarnings("java:S125") // confusing comments with codes
+ private static String[] splitDeviceIdString(String deviceIdString) {
+ List<String> splits = Arrays.asList(PathNodesGenerator.splitPathToNodes(deviceIdString));
+ int segmentCnt = splits.size();
+
+ String tableName;
+ String[] segments;
+ // assuming DEFAULT_SEGMENT_NUM_FOR_TABLE_NAME = 3
+ if (segmentCnt == 1) {
+ // "root" -> {"root"}
+ segments = new String[1];
+ segments[0] = splits.get(0);
+ } else if (segmentCnt < TSFileConfig.DEFAULT_SEGMENT_NUM_FOR_TABLE_NAME + 1) {
+ // "root.a" -> {"root", "a"}
+ // "root.a.b" -> {"root.a", "b"}
+ tableName =
+ segmentCnt == 1 ? "" : String.join(PATH_SEPARATOR, splits.subList(0, segmentCnt - 1));
+ segments = new String[2];
+ segments[0] = tableName;
+ segments[1] = splits.get(segmentCnt - 1);
+ } else {
+ // "root.a.b.c" -> {"root.a.b", "c"}
+ // "root.a.b.c.d" -> {"root.a.b", "c", "d"}
+ tableName =
+ String.join(
+ PATH_SEPARATOR, splits.subList(0, TSFileConfig.DEFAULT_SEGMENT_NUM_FOR_TABLE_NAME));
+ String[] idSegments =
+ splits
+ .subList(TSFileConfig.DEFAULT_SEGMENT_NUM_FOR_TABLE_NAME, splits.size())
+ .toArray(new String[0]);
+ segments = new String[idSegments.length + 1];
+ segments[0] = tableName;
+ System.arraycopy(idSegments, 0, segments, 1, idSegments.length);
+ }
+
+ return segments;
+ }
+
+ public static Deserializer getDESERIALIZER() {
+ return DESERIALIZER;
+ }
+
+ public static Factory getFACTORY() {
+ return FACTORY;
+ }
+
+ @Override
+ public int serialize(ByteBuffer byteBuffer) {
+ int cnt = 0;
+ cnt += ReadWriteForEncodingUtils.writeUnsignedVarInt(segments.length, byteBuffer);
+ for (String segment : segments) {
+ cnt += ReadWriteIOUtils.writeVar(segment, byteBuffer);
+ }
+ return cnt;
+ }
+
+ @Override
+ public int serialize(OutputStream outputStream) throws IOException {
+ int cnt = 0;
+ cnt += ReadWriteForEncodingUtils.writeUnsignedVarInt(segments.length, outputStream);
+ for (String segment : segments) {
+ cnt += ReadWriteIOUtils.writeVar(segment, outputStream);
+ }
+ return cnt;
+ }
+
+ public static StringArrayDeviceID deserialize(ByteBuffer byteBuffer) {
+ final int cnt = ReadWriteForEncodingUtils.readUnsignedVarInt(byteBuffer);
+ if (cnt == 0) {
+ return new StringArrayDeviceID(new String[] {""});
+ }
+
+ String[] segments = new String[cnt];
+ for (int i = 0; i < cnt; i++) {
+ segments[i] = ReadWriteIOUtils.readVarIntString(byteBuffer);
+ }
+ return new StringArrayDeviceID(segments);
+ }
+
+ public static StringArrayDeviceID deserialize(InputStream stream) throws IOException {
+ final int cnt = ReadWriteForEncodingUtils.readUnsignedVarInt(stream);
+ if (cnt == 0) {
+ return new StringArrayDeviceID(new String[] {""});
+ }
+
+ String[] segments = new String[cnt];
+ for (int i = 0; i < cnt; i++) {
+ segments[i] = ReadWriteIOUtils.readVarIntString(stream);
+ }
+ return new StringArrayDeviceID(segments);
+ }
+
+ @Override
+ public byte[] getBytes() {
+ ByteArrayOutputStream publicBAOS = new ByteArrayOutputStream(256);
+ try {
+ serialize(publicBAOS);
+ } catch (IOException e) {
+ throw new TsFileRuntimeException(e);
+ }
+ return publicBAOS.toByteArray();
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return segments == null || segments.length == 0;
+ }
+
+ @Override
+ public String getTableName() {
+ return segments[0];
+ }
+
+ @Override
+ public int segmentNum() {
+ return segments.length;
+ }
+
+ @Override
+ public String segment(int i) {
+ return segments[i];
+ }
+
+ @Override
+ public int compareTo(IDeviceID o) {
+ int thisSegmentNum = segmentNum();
+ int otherSegmentNum = o.segmentNum();
+ for (int i = 0; i < thisSegmentNum; i++) {
+ if (i >= otherSegmentNum) {
+ // the other ID is a prefix of this one
+ return 1;
+ }
+ final int comp =
+ Objects.compare(this.segment(i), ((String) o.segment(i)), WriteUtils::compareStrings);
+ if (comp != 0) {
+ // the partial comparison has a result
+ return comp;
+ }
+ }
+
+ if (thisSegmentNum < otherSegmentNum) {
+ // this ID is a prefix of the other one
+ return -1;
+ }
+
+ // two ID equal
+ return 0;
+ }
+
+ @Override
+ public long ramBytesUsed() {
+ return INSTANCE_SIZE + RamUsageEstimator.sizeOf(segments);
+ }
+
+ @Override
+ public int serializedSize() {
+ if (serializedSize != -1) {
+ return serializedSize;
+ }
+
+ int cnt = ReadWriteForEncodingUtils.varIntSize(segments.length);
+ for (String segment : segments) {
+ if (segment != null) {
+ byte[] bytes = segment.getBytes(TSFileConfig.STRING_CHARSET);
+ cnt += ReadWriteForEncodingUtils.varIntSize(bytes.length);
+ cnt += bytes.length;
+ } else {
+ cnt += ReadWriteForEncodingUtils.varIntSize(ReadWriteIOUtils.NO_BYTE_TO_READ);
+ }
+ }
+ serializedSize = cnt;
+ return cnt;
+ }
+
+ @Override
+ public String toString() {
+ return String.join(".", segments);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ StringArrayDeviceID deviceID = (StringArrayDeviceID) o;
+ return Objects.deepEquals(segments, deviceID.segments);
+ }
+
+ @Override
+ public int hashCode() {
+ return Arrays.hashCode(segments);
+ }
+
+ @Override
+ public String[] getSegments() {
+ return segments;
+ }
+}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchema.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchema.java
new file mode 100644
index 00000000..474c4a03
--- /dev/null
+++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchema.java
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tsfile.file.metadata;
+
+import org.apache.tsfile.compatibility.DeserializeConfig;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.utils.ReadWriteForEncodingUtils;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+import org.apache.tsfile.write.record.Tablet.ColumnType;
+import org.apache.tsfile.write.schema.IMeasurementSchema;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+public class TableSchema {
+ // the tableName is not serialized since the TableSchema is always stored in a Map, from whose
+ // key the tableName can be known
+ protected String tableName;
+ protected List<IMeasurementSchema> columnSchemas;
+ protected List<ColumnType> columnTypes;
+ protected boolean updatable = false;
+
+ // columnName -> pos in columnSchemas;
+ private Map<String, Integer> columnPosIndex;
+
+ public TableSchema(String tableName) {
+ this.tableName = tableName;
+ this.columnSchemas = new ArrayList<>();
+ this.columnTypes = new ArrayList<>();
+ this.updatable = true;
+ }
+
+ public TableSchema(
+ String tableName, List<IMeasurementSchema> columnSchemas, List<ColumnType> columnTypes) {
+ this.tableName = tableName;
+ this.columnSchemas = columnSchemas;
+ this.columnTypes = columnTypes;
+ }
+
+ public Map<String, Integer> getColumnPosIndex() {
+ if (columnPosIndex == null) {
+ columnPosIndex = new HashMap<>();
+ }
+ return columnPosIndex;
+ }
+
+ public int findColumnIndex(String columnName) {
+ return getColumnPosIndex()
+ .computeIfAbsent(
+ columnName,
+ colName -> {
+ for (int i = 0; i < columnSchemas.size(); i++) {
+ if (columnSchemas.get(i).getMeasurementId().equals(columnName)) {
+ return i;
+ }
+ }
+ return -1;
+ });
+ }
+
+ public IMeasurementSchema findColumnSchema(String columnName) {
+ final int columnIndex = findColumnIndex(columnName);
+ return columnIndex >= 0 ? columnSchemas.get(columnIndex) : null;
+ }
+
+ public void update(ChunkGroupMetadata chunkGroupMetadata) {
+ if (!updatable) {
+ return;
+ }
+
+ for (ChunkMetadata chunkMetadata : chunkGroupMetadata.getChunkMetadataList()) {
+ int columnIndex = findColumnIndex(chunkMetadata.getMeasurementUid());
+ // if the measurement is not found in the column list, add it
+ if (columnIndex == -1) {
+ columnSchemas.add(chunkMetadata.toMeasurementSchema());
+ columnTypes.add(ColumnType.MEASUREMENT);
+ getColumnPosIndex().put(chunkMetadata.getMeasurementUid(), columnSchemas.size() - 1);
+ } else {
+ final IMeasurementSchema originSchema = columnSchemas.get(columnIndex);
+ if (originSchema.getType() != chunkMetadata.getDataType()) {
+ originSchema.setType(TSDataType.STRING);
+ }
+ }
+ }
+ }
+
+ public List<IMeasurementSchema> getColumnSchemas() {
+ return columnSchemas;
+ }
+
+ public List<ColumnType> getColumnTypes() {
+ return columnTypes;
+ }
+
+ public int serialize(OutputStream out) throws IOException {
+ int cnt = 0;
+ if (columnSchemas != null) {
+ cnt += ReadWriteForEncodingUtils.writeUnsignedVarInt(columnSchemas.size(), out);
+ for (int i = 0; i < columnSchemas.size(); i++) {
+ IMeasurementSchema columnSchema = columnSchemas.get(i);
+ ColumnType columnType = columnTypes.get(i);
+ cnt += columnSchema.serializeTo(out);
+ cnt += ReadWriteIOUtils.write(columnType.ordinal(), out);
+ }
+ } else {
+ cnt += ReadWriteForEncodingUtils.writeUnsignedVarInt(0, out);
+ }
+
+ return cnt;
+ }
+
+ public int serializedSize() {
+ try {
+ return serialize(new ByteArrayOutputStream());
+ } catch (IOException e) {
+ return -1;
+ }
+ }
+
+ public static TableSchema deserialize(ByteBuffer buffer, DeserializeConfig context) {
+ final int columnCnt = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer);
+ List<IMeasurementSchema> measurementSchemas = new ArrayList<>(columnCnt);
+ List<ColumnType> columnTypes = new ArrayList<>();
+ for (int i = 0; i < columnCnt; i++) {
+ MeasurementSchema measurementSchema =
+ context.measurementSchemaBufferDeserializer.deserialize(buffer, context);
+ measurementSchemas.add(measurementSchema);
+ columnTypes.add(ColumnType.values()[buffer.getInt()]);
+ }
+ return new TableSchema(null, measurementSchemas, columnTypes);
+ }
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ public void setTableName(String tableName) {
+ this.tableName = tableName;
+ }
+
+ @Override
+ public String toString() {
+ return "TableSchema{"
+ + "tableName='"
+ + tableName
+ + '\''
+ + ", columnSchemas="
+ + columnSchemas
+ + ", columnTypes="
+ + columnTypes
+ + '}';
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof TableSchema)) {
+ return false;
+ }
+ TableSchema that = (TableSchema) o;
+ return Objects.equals(tableName, that.tableName)
+ && Objects.equals(columnSchemas, that.columnSchemas)
+ && Objects.equals(columnTypes, that.columnTypes);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(tableName, columnSchemas, columnTypes);
+ }
+}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TimeseriesMetadata.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TimeseriesMetadata.java
index 17a326e0..e59fd9dc 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TimeseriesMetadata.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TimeseriesMetadata.java
@@ -19,6 +19,7 @@
package org.apache.tsfile.file.metadata;
+import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.metadata.statistics.Statistics;
import org.apache.tsfile.read.controller.IChunkMetadataLoader;
@@ -356,6 +357,21 @@ public class TimeseriesMetadata implements ITimeSeriesMetadata {
return retainedSize;
}
+ public int serializedSizeWithoutMetadata() {
+ final byte[] bytes = measurementId.getBytes(TSFileConfig.STRING_CHARSET);
+ return Byte.BYTES
+ + // metadata type
+ ReadWriteForEncodingUtils.varIntSize(bytes.length)
+ + // measurementId size
+ bytes.length
+ + // measurmentId bytes
+ Byte.BYTES
+ + // data type
+ ReadWriteForEncodingUtils.uVarIntSize(chunkMetaDataListDataSize)
+ + // ChunkMetadata num
+ statistics.getSerializedSize(); // statistic
+ }
+
@Override
public String toString() {
return "TimeseriesMetadata{"
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TsFileMetadata.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TsFileMetadata.java
index 68bd6dd6..ec3878c2 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TsFileMetadata.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TsFileMetadata.java
@@ -19,6 +19,7 @@
package org.apache.tsfile.file.metadata;
+import org.apache.tsfile.compatibility.DeserializeConfig;
import org.apache.tsfile.utils.BloomFilter;
import org.apache.tsfile.utils.ReadWriteForEncodingUtils;
import org.apache.tsfile.utils.ReadWriteIOUtils;
@@ -26,6 +27,10 @@ import org.apache.tsfile.utils.ReadWriteIOUtils;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.TreeMap;
/** TSFileMetaData collects all metadata info and saves in its data structure. */
public class TsFileMetadata {
@@ -34,10 +39,14 @@ public class TsFileMetadata {
private BloomFilter bloomFilter;
// List of <name, offset, childMetadataIndexType>
- private MetadataIndexNode metadataIndex;
+ private Map<String, MetadataIndexNode> tableMetadataIndexNodeMap;
+ private Map<String, TableSchema> tableSchemaMap;
+ private Map<String, String> tsFileProperties;
// offset of MetaMarker.SEPARATOR
private long metaOffset;
+ // offset from MetaMarker.SEPARATOR (exclusive) to tsFileProperties
+ private int propertiesOffset;
/**
* deserialize data from the buffer.
@@ -45,11 +54,31 @@ public class TsFileMetadata {
* @param buffer -buffer use to deserialize
* @return -a instance of TsFileMetaData
*/
- public static TsFileMetadata deserializeFrom(ByteBuffer buffer) {
+ public static TsFileMetadata deserializeFrom(ByteBuffer buffer, DeserializeConfig context) {
TsFileMetadata fileMetaData = new TsFileMetadata();
+ int startPos = buffer.position();
// metadataIndex
- fileMetaData.metadataIndex = MetadataIndexNode.deserializeFrom(buffer, true);
+ int tableIndexNodeNum = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer);
+ Map<String, MetadataIndexNode> tableIndexNodeMap = new TreeMap<>();
+ for (int i = 0; i < tableIndexNodeNum; i++) {
+ String tableName = ReadWriteIOUtils.readVarIntString(buffer);
+ MetadataIndexNode metadataIndexNode =
+ context.deviceMetadataIndexNodeBufferDeserializer.deserialize(buffer, context);
+ tableIndexNodeMap.put(tableName, metadataIndexNode);
+ }
+ fileMetaData.setTableMetadataIndexNodeMap(tableIndexNodeMap);
+
+ // tableSchemas
+ int tableSchemaNum = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer);
+ Map<String, TableSchema> tableSchemaMap = new HashMap<>();
+ for (int i = 0; i < tableSchemaNum; i++) {
+ String tableName = ReadWriteIOUtils.readVarIntString(buffer);
+ TableSchema tableSchema = context.tableSchemaBufferDeserializer.deserialize(buffer, context);
+ tableSchema.setTableName(tableName);
+ tableSchemaMap.put(tableName, tableSchema);
+ }
+ fileMetaData.setTableSchemaMap(tableSchemaMap);
// metaOffset
long metaOffset = ReadWriteIOUtils.readLong(buffer);
@@ -58,9 +87,25 @@ public class TsFileMetadata {
// read bloom filter
if (buffer.hasRemaining()) {
byte[] bytes = ReadWriteIOUtils.readByteBufferWithSelfDescriptionLength(buffer);
- int filterSize = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer);
- int hashFunctionSize = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer);
- fileMetaData.bloomFilter = BloomFilter.buildBloomFilter(bytes, filterSize, hashFunctionSize);
+ if (bytes.length != 0) {
+ int filterSize = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer);
+ int hashFunctionSize = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer);
+ fileMetaData.bloomFilter =
+ BloomFilter.buildBloomFilter(bytes, filterSize, hashFunctionSize);
+ }
+ }
+
+ fileMetaData.propertiesOffset = buffer.position() - startPos;
+
+ if (buffer.hasRemaining()) {
+ int propertiesSize = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer);
+ Map<String, String> propertiesMap = new HashMap<>();
+ for (int i = 0; i < propertiesSize; i++) {
+ String key = ReadWriteIOUtils.readVarIntString(buffer);
+ String value = ReadWriteIOUtils.readVarIntString(buffer);
+ propertiesMap.put(key, value);
+ }
+ fileMetaData.tsFileProperties = propertiesMap;
}
return fileMetaData;
@@ -84,15 +129,45 @@ public class TsFileMetadata {
public int serializeTo(OutputStream outputStream) throws IOException {
int byteLen = 0;
- // metadataIndex
- if (metadataIndex != null) {
- byteLen += metadataIndex.serializeTo(outputStream);
+ if (tableMetadataIndexNodeMap != null) {
+ byteLen +=
+ ReadWriteForEncodingUtils.writeUnsignedVarInt(
+ tableMetadataIndexNodeMap.size(), outputStream);
+ for (Entry<String, MetadataIndexNode> entry : tableMetadataIndexNodeMap.entrySet()) {
+ byteLen += ReadWriteIOUtils.writeVar(entry.getKey(), outputStream);
+ byteLen += entry.getValue().serializeTo(outputStream);
+ }
} else {
- byteLen += ReadWriteIOUtils.write(0, outputStream);
+ byteLen += ReadWriteForEncodingUtils.writeUnsignedVarInt(0, outputStream);
+ }
+
+ if (tableSchemaMap != null) {
+ byteLen += ReadWriteForEncodingUtils.writeUnsignedVarInt(tableSchemaMap.size(), outputStream);
+ for (Entry<String, TableSchema> entry : tableSchemaMap.entrySet()) {
+ byteLen += ReadWriteIOUtils.writeVar(entry.getKey(), outputStream);
+ byteLen += entry.getValue().serialize(outputStream);
+ }
+ } else {
+ byteLen += ReadWriteForEncodingUtils.writeUnsignedVarInt(0, outputStream);
}
// metaOffset
byteLen += ReadWriteIOUtils.write(metaOffset, outputStream);
+ if (bloomFilter != null) {
+ byteLen += serializeBloomFilter(outputStream, bloomFilter);
+ } else {
+ byteLen += ReadWriteIOUtils.write(0, outputStream);
+ }
+
+ byteLen +=
+ ReadWriteForEncodingUtils.writeVarInt(
+ tsFileProperties != null ? tsFileProperties.size() : 0, outputStream);
+ if (tsFileProperties != null) {
+ for (Entry<String, String> entry : tsFileProperties.entrySet()) {
+ byteLen += ReadWriteIOUtils.writeVar(entry.getKey(), outputStream);
+ byteLen += ReadWriteIOUtils.writeVar(entry.getValue(), outputStream);
+ }
+ }
return byteLen;
}
@@ -118,11 +193,28 @@ public class TsFileMetadata {
this.metaOffset = metaOffset;
}
- public MetadataIndexNode getMetadataIndex() {
- return metadataIndex;
+ public void setTableMetadataIndexNodeMap(
+ Map<String, MetadataIndexNode> tableMetadataIndexNodeMap) {
+ this.tableMetadataIndexNodeMap = tableMetadataIndexNodeMap;
+ }
+
+ public void setTableSchemaMap(Map<String, TableSchema> tableSchemaMap) {
+ this.tableSchemaMap = tableSchemaMap;
+ }
+
+ public Map<String, MetadataIndexNode> getTableMetadataIndexNodeMap() {
+ return tableMetadataIndexNodeMap;
+ }
+
+ public MetadataIndexNode getTableMetadataIndexNode(String tableName) {
+ MetadataIndexNode metadataIndexNode = tableMetadataIndexNodeMap.get(tableName);
+ if (metadataIndexNode == null) {
+ metadataIndexNode = tableMetadataIndexNodeMap.get("");
+ }
+ return metadataIndexNode;
}
- public void setMetadataIndex(MetadataIndexNode metadataIndex) {
- this.metadataIndex = metadataIndex;
+ public Map<String, TableSchema> getTableSchemaMap() {
+ return tableSchemaMap;
}
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java
index fa9b8ab3..b93dbb3d 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java
@@ -22,12 +22,15 @@ package org.apache.tsfile.read;
import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.common.conf.TSFileDescriptor;
import org.apache.tsfile.common.constant.TsFileConstant;
+import org.apache.tsfile.compatibility.CompatibilityUtils;
+import org.apache.tsfile.compatibility.DeserializeConfig;
import org.apache.tsfile.compress.IUnCompressor;
import org.apache.tsfile.encoding.decoder.Decoder;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.exception.StopReadTsFileByInterruptException;
import org.apache.tsfile.exception.TsFileRuntimeException;
import org.apache.tsfile.exception.TsFileStatisticsMistakesException;
+import org.apache.tsfile.exception.read.FileVersionTooOldException;
import org.apache.tsfile.file.IMetadataIndexEntry;
import org.apache.tsfile.file.MetaMarker;
import org.apache.tsfile.file.header.ChunkGroupHeader;
@@ -43,7 +46,6 @@ import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.file.metadata.ITimeSeriesMetadata;
import org.apache.tsfile.file.metadata.MeasurementMetadataIndexEntry;
import org.apache.tsfile.file.metadata.MetadataIndexNode;
-import org.apache.tsfile.file.metadata.PlainDeviceID;
import org.apache.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.tsfile.file.metadata.TsFileMetadata;
import org.apache.tsfile.file.metadata.enums.CompressionType;
@@ -67,6 +69,7 @@ import org.apache.tsfile.utils.ReadWriteIOUtils;
import org.apache.tsfile.utils.TsPrimitiveType;
import org.apache.tsfile.write.schema.IMeasurementSchema;
import org.apache.tsfile.write.schema.MeasurementSchema;
+import org.apache.tsfile.write.schema.Schema;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -79,12 +82,12 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.NoSuchElementException;
import java.util.Objects;
import java.util.Queue;
@@ -120,6 +123,10 @@ public class TsFileSequenceReader implements AutoCloseable {
private long minPlanIndex = Long.MAX_VALUE;
private long maxPlanIndex = Long.MIN_VALUE;
+ private byte fileVersion;
+
+ private DeserializeConfig deserializeConfig = new DeserializeConfig();
+
/**
* Create a file reader of the given file. The reader will read the tail of the file to get the
* file metadata size.Then the reader will skip the first
@@ -208,7 +215,32 @@ public class TsFileSequenceReader implements AutoCloseable {
this.fileMetadataSize = fileMetadataSize;
}
+ private void loadFileVersion() throws IOException {
+ tsFileInput.position(TSFileConfig.MAGIC_STRING.getBytes(TSFileConfig.STRING_CHARSET).length);
+ final ByteBuffer buffer = ByteBuffer.allocate(1);
+ tsFileInput.read(buffer);
+ buffer.flip();
+ fileVersion = buffer.get();
+
+ checkFileVersion();
+ configDeserializer();
+ }
+
+ private void configDeserializer() throws IOException {
+ if (fileVersion == TSFileConfig.VERSION_NUMBER_V3) {
+ deserializeConfig = CompatibilityUtils.v3DeserializeConfig;
+ }
+ }
+
+ private void checkFileVersion() throws FileVersionTooOldException {
+ if (TSFileConfig.VERSION_NUMBER - fileVersion > 1) {
+ throw new FileVersionTooOldException(fileVersion, (byte) (TSFileConfig.VERSION_NUMBER - 1));
+ }
+ }
+
public void loadMetadataSize() throws IOException {
+ loadFileVersion();
+
ByteBuffer metadataSize = ByteBuffer.allocate(Integer.BYTES);
if (readTailMagic().equals(TSFileConfig.MAGIC_STRING)) {
tsFileInput.read(
@@ -299,7 +331,8 @@ public class TsFileSequenceReader implements AutoCloseable {
synchronized (this) {
if (tsFileMetaData == null) {
tsFileMetaData =
- TsFileMetadata.deserializeFrom(readData(fileMetadataPos, fileMetadataSize));
+ deserializeConfig.tsFileMetadataBufferDeserializer.deserialize(
+ readData(fileMetadataPos, fileMetadataSize), deserializeConfig);
}
}
}
@@ -376,7 +409,8 @@ public class TsFileSequenceReader implements AutoCloseable {
public TimeseriesMetadata readTimeseriesMetadata(
IDeviceID device, String measurement, boolean ignoreNotExists) throws IOException {
readFileMetadata();
- MetadataIndexNode deviceMetadataIndexNode = tsFileMetaData.getMetadataIndex();
+ MetadataIndexNode deviceMetadataIndexNode =
+ tsFileMetaData.getTableMetadataIndexNode(device.getTableName());
Pair<IMetadataIndexEntry, Long> metadataIndexPair =
getMetadataAndEndOffsetOfDeviceNode(deviceMetadataIndexNode, device, true);
if (metadataIndexPair == null) {
@@ -389,7 +423,9 @@ public class TsFileSequenceReader implements AutoCloseable {
MetadataIndexNode metadataIndexNode = deviceMetadataIndexNode;
if (!metadataIndexNode.getNodeType().equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) {
try {
- metadataIndexNode = MetadataIndexNode.deserializeFrom(buffer, false);
+ metadataIndexNode =
+ deserializeConfig.measurementMetadataIndexNodeBufferDeserializer.deserialize(
+ buffer, deserializeConfig);
} catch (Exception e) {
logger.error(METADATA_INDEX_NODE_DESERIALIZE_ERROR, file);
throw e;
@@ -436,26 +472,29 @@ public class TsFileSequenceReader implements AutoCloseable {
public ITimeSeriesMetadata readITimeseriesMetadata(Path path, boolean ignoreNotExists)
throws IOException {
readFileMetadata();
- MetadataIndexNode deviceMetadataIndexNode = tsFileMetaData.getMetadataIndex();
+ MetadataIndexNode deviceMetadataIndexNode =
+ tsFileMetaData.getTableMetadataIndexNode(path.getIDeviceID().getTableName());
Pair<IMetadataIndexEntry, Long> metadataIndexPair =
getMetadataAndEndOffsetOfDeviceNode(deviceMetadataIndexNode, path.getIDeviceID(), true);
if (metadataIndexPair == null) {
if (ignoreNotExists) {
return null;
}
- throw new IOException("Device {" + path.getDevice() + "} is not in tsFileMetaData");
+ throw new IOException("Device {" + path.getDeviceString() + "} is not in tsFileMetaData");
}
ByteBuffer buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right);
MetadataIndexNode metadataIndexNode;
TimeseriesMetadata firstTimeseriesMetadata;
try {
// next layer MeasurementNode of the specific DeviceNode
- metadataIndexNode = MetadataIndexNode.deserializeFrom(buffer, false);
+ metadataIndexNode =
+ deserializeConfig.measurementMetadataIndexNodeBufferDeserializer.deserialize(
+ buffer, deserializeConfig);
} catch (Exception e) {
logger.error(METADATA_INDEX_NODE_DESERIALIZE_ERROR, file);
throw e;
}
- firstTimeseriesMetadata = tryToGetFirstTimeseriesMetadata(metadataIndexNode);
+ firstTimeseriesMetadata = getTimeColumnMetadata(metadataIndexNode);
metadataIndexPair =
getMetadataAndEndOffsetOfMeasurementNode(metadataIndexNode, path.getMeasurement(), false);
@@ -543,7 +582,8 @@ public class TsFileSequenceReader implements AutoCloseable {
private Pair<IMetadataIndexEntry, Long> getLeafMetadataIndexPair(
IDeviceID device, String measurement) throws IOException {
readFileMetadata();
- MetadataIndexNode deviceMetadataIndexNode = tsFileMetaData.getMetadataIndex();
+ MetadataIndexNode deviceMetadataIndexNode =
+ tsFileMetaData.getTableMetadataIndexNode(device.getTableName());
Pair<IMetadataIndexEntry, Long> metadataIndexPair =
getMetadataAndEndOffsetOfDeviceNode(deviceMetadataIndexNode, device, true);
if (metadataIndexPair == null) {
@@ -553,7 +593,9 @@ public class TsFileSequenceReader implements AutoCloseable {
MetadataIndexNode metadataIndexNode = deviceMetadataIndexNode;
if (!metadataIndexNode.getNodeType().equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) {
try {
- metadataIndexNode = MetadataIndexNode.deserializeFrom(buffer, false);
+ metadataIndexNode =
+ deserializeConfig.measurementMetadataIndexNodeBufferDeserializer.deserialize(
+ buffer, deserializeConfig);
} catch (Exception e) {
logger.error(METADATA_INDEX_NODE_DESERIALIZE_ERROR, file);
throw e;
@@ -564,84 +606,192 @@ public class TsFileSequenceReader implements AutoCloseable {
return metadataIndexPair;
}
- // This method is only used for TsFile
- public List<ITimeSeriesMetadata> readITimeseriesMetadata(
- IDeviceID device, Set<String> measurements) throws IOException {
+ private MetadataIndexNode getTableRootNode(String tableName) throws IOException {
+ MetadataIndexNode metadataIndexNode = tsFileMetaData.getTableMetadataIndexNode(tableName);
+ if (metadataIndexNode == null && fileVersion < TSFileConfig.VERSION_NUMBER) {
+ // this file if from an old version, and all its metadata should have an anonymous root
+ metadataIndexNode = tsFileMetaData.getTableMetadataIndexNode("");
+ }
+ return metadataIndexNode;
+ }
+
+ /**
+ * Searching from the start node and try to find the root node of the deviceID.
+ *
+ * @param deviceID desired device
+ * @param startNode start of the search, if not provided, start from the table root
+ * @return MetadataIndexNode which is the root of deviceID
+ */
+ private MetadataIndexNode getDeviceRootNode(IDeviceID deviceID, MetadataIndexNode startNode)
+ throws IOException {
readFileMetadata();
- MetadataIndexNode deviceMetadataIndexNode = tsFileMetaData.getMetadataIndex();
- Pair<IMetadataIndexEntry, Long> metadataIndexPair =
- getMetadataAndEndOffsetOfDeviceNode(deviceMetadataIndexNode, device, false);
- if (metadataIndexPair == null) {
- return Collections.emptyList();
+ startNode = startNode != null ? startNode : getTableRootNode(deviceID.getTableName());
+ if (startNode == null) {
+ return null;
}
- List<ITimeSeriesMetadata> resultTimeseriesMetadataList = new ArrayList<>();
- List<String> measurementList = new ArrayList<>(measurements);
- Set<String> measurementsHadFound = new HashSet<>();
- // the content of next Layer MeasurementNode of the specific device's DeviceNode
- ByteBuffer buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right);
- Pair<IMetadataIndexEntry, Long> measurementMetadataIndexPair = metadataIndexPair;
- List<TimeseriesMetadata> timeseriesMetadataList = new ArrayList<>();
- // next layer MeasurementNode of the specific DeviceNode
MetadataIndexNode measurementMetadataIndexNode;
- try {
- measurementMetadataIndexNode = MetadataIndexNode.deserializeFrom(buffer, false);
- } catch (Exception e) {
- logger.error(METADATA_INDEX_NODE_DESERIALIZE_ERROR, file);
- throw e;
+ ByteBuffer buffer;
+ if (startNode.isDeviceLevel()) {
+ Pair<IMetadataIndexEntry, Long> metadataIndexPair =
+ getMetadataAndEndOffsetOfDeviceNode(startNode, deviceID, true);
+ if (metadataIndexPair == null) {
+ return null;
+ }
+
+ // the content of next Layer MeasurementNode of the specific device's DeviceNode
+ buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right);
+ // next layer MeasurementNode of the specific DeviceNode
+ try {
+ measurementMetadataIndexNode =
+ deserializeConfig.measurementMetadataIndexNodeBufferDeserializer.deserialize(
+ buffer, deserializeConfig);
+ } catch (Exception e) {
+ logger.error(METADATA_INDEX_NODE_DESERIALIZE_ERROR, file);
+ throw e;
+ }
+ } else {
+ measurementMetadataIndexNode = startNode;
}
- // Get the first timeseriesMetadata of the device
- TimeseriesMetadata firstTimeseriesMetadata =
- tryToGetFirstTimeseriesMetadata(measurementMetadataIndexNode);
+ return measurementMetadataIndexNode;
+ }
- for (int i = 0; i < measurementList.size(); i++) {
- if (measurementsHadFound.contains(measurementList.get(i))) {
- continue;
+ /**
+ * Read the TimeseriesMetadata of the given measurement and its successors from the index node
+ * into the list.
+ *
+ * @param timeseriesMetadataList result holder
+ * @param node index node to be read from
+ * @param measurement the desired measurement
+ * @return true if the measurement exists
+ * @throws IOException when read fails
+ */
+ public boolean readITimeseriesMetadata(
+ List<TimeseriesMetadata> timeseriesMetadataList, MetadataIndexNode node, String measurement)
+ throws IOException {
+ Pair<IMetadataIndexEntry, Long> measurementMetadataIndexPair =
+ getMetadataAndEndOffsetOfMeasurementNode(node, measurement, false);
+
+ if (measurementMetadataIndexPair == null) {
+ return false;
+ }
+ // the content of TimeseriesNode of the specific MeasurementLeafNode
+ ByteBuffer buffer =
+ readData(measurementMetadataIndexPair.left.getOffset(), measurementMetadataIndexPair.right);
+ while (buffer.hasRemaining()) {
+ try {
+ timeseriesMetadataList.add(TimeseriesMetadata.deserializeFrom(buffer, true));
+ } catch (Exception e) {
+ logger.error(
+ "Something error happened while deserializing TimeseriesMetadata of file {}", file);
+ throw e;
}
- timeseriesMetadataList.clear();
- measurementMetadataIndexPair =
- getMetadataAndEndOffsetOfMeasurementNode(
- measurementMetadataIndexNode, measurementList.get(i), false);
+ }
+ return true;
+ }
+
+ /**
+ * Read TimeSeriesMetadata of the given device. This method is only used for TsFile.
+ *
+ * @param device deviceId to be read
+ * @param measurements measurements to be read
+ * @param root search start node, if not provided, use the root node of the table of the device
+ * @param mergeAlignedSeries see @return
+ * @return when the device is not aligned, or mergeAlignedSeries is false, each result correspond
+ * to one series in the provided measurements (if exists); otherwise, all columns in the
+ * aligned device will be merged into one AlignedTimeSeriesMetadata.
+ * @throws IOException if read fails
+ */
+ public List<ITimeSeriesMetadata> readITimeseriesMetadata(
+ IDeviceID device,
+ Set<String> measurements,
+ MetadataIndexNode root,
+ boolean mergeAlignedSeries)
+ throws IOException {
+ // find the index node associated with the device
+ final MetadataIndexNode measurementMetadataIndexNode = getDeviceRootNode(device, root);
+ if (measurementMetadataIndexNode == null) {
+ return Collections.emptyList();
+ }
+
+ // Get the time column metadata if the device is aligned
+ TimeseriesMetadata timeColumnMetadata = getTimeColumnMetadata(measurementMetadataIndexNode);
+ List<TimeseriesMetadata> valueTimeseriesMetadataList =
+ timeColumnMetadata != null ? new ArrayList<>() : null;
+
+ List<ITimeSeriesMetadata> resultTimeseriesMetadataList = new ArrayList<>();
+ List<String> measurementList = new ArrayList<>(measurements);
+ measurementList.sort(null);
+ boolean[] measurementFound = new boolean[measurements.size()];
+ int measurementFoundCnt = 0;
- if (measurementMetadataIndexPair == null) {
+ List<TimeseriesMetadata> timeseriesMetadataList = new ArrayList<>();
+ for (int i = 0;
+ i < measurementList.size() && measurementFoundCnt < measurementList.size();
+ i++) {
+ final String measurementName = measurementList.get(i);
+ timeseriesMetadataList.clear();
+ // read the leaf node that may contain the i-th measurement into a list
+ if (measurementFound[i]
+ || !readITimeseriesMetadata(
+ timeseriesMetadataList, measurementMetadataIndexNode, measurementName)) {
continue;
}
- // the content of TimeseriesNode of the specific MeasurementLeafNode
- buffer =
- readData(
- measurementMetadataIndexPair.left.getOffset(), measurementMetadataIndexPair.right);
- while (buffer.hasRemaining()) {
- try {
- timeseriesMetadataList.add(TimeseriesMetadata.deserializeFrom(buffer, true));
- } catch (Exception e) {
- logger.error(
- "Something error happened while deserializing TimeseriesMetadata of file {}", file);
- throw e;
- }
+ // in the list, search for the all measurements that are not found
+ measurementFoundCnt +=
+ searchInTimeseriesList(
+ measurementList,
+ i,
+ measurementFound,
+ timeseriesMetadataList,
+ resultTimeseriesMetadataList,
+ timeColumnMetadata,
+ valueTimeseriesMetadataList,
+ mergeAlignedSeries);
+ }
+ if (valueTimeseriesMetadataList != null && !valueTimeseriesMetadataList.isEmpty()) {
+ resultTimeseriesMetadataList.add(
+ new AlignedTimeSeriesMetadata(timeColumnMetadata, valueTimeseriesMetadataList));
+ }
+ return resultTimeseriesMetadataList;
+ }
+
+ private int searchInTimeseriesList(
+ List<String> measurementList,
+ int startIndex,
+ boolean[] measurementFound,
+ List<TimeseriesMetadata> timeseriesMetadataList,
+ List<ITimeSeriesMetadata> resultTimeseriesMetadataList,
+ TimeseriesMetadata timeColumnMetadata,
+ List<TimeseriesMetadata> valueTimeseriesMetadataList,
+ boolean mergeAlignedSeries) {
+ int numOfFoundMeasurements = 0;
+ for (int j = startIndex; j < measurementList.size(); j++) {
+ int searchResult;
+ if (measurementFound[j]
+ || (searchResult =
+ binarySearchInTimeseriesMetadataList(
+ timeseriesMetadataList, measurementList.get(j)))
+ < 0) {
+ continue;
}
- for (int j = i; j < measurementList.size(); j++) {
- String current = measurementList.get(j);
- if (!measurementsHadFound.contains(current)) {
- int searchResult = binarySearchInTimeseriesMetadataList(timeseriesMetadataList, current);
- if (searchResult >= 0) {
- if (firstTimeseriesMetadata != null) {
- List<TimeseriesMetadata> valueTimeseriesMetadataList = new ArrayList<>();
- valueTimeseriesMetadataList.add(timeseriesMetadataList.get(searchResult));
- resultTimeseriesMetadataList.add(
- new AlignedTimeSeriesMetadata(
- firstTimeseriesMetadata, valueTimeseriesMetadataList));
- } else {
- resultTimeseriesMetadataList.add(timeseriesMetadataList.get(searchResult));
- }
- measurementsHadFound.add(current);
- }
- }
- if (measurementsHadFound.size() == measurements.size()) {
- return resultTimeseriesMetadataList;
+
+ final TimeseriesMetadata valueColumnMetadata = timeseriesMetadataList.get(searchResult);
+ if (timeColumnMetadata != null) {
+ if (!mergeAlignedSeries) {
+ resultTimeseriesMetadataList.add(
+ new AlignedTimeSeriesMetadata(
+ timeColumnMetadata, Collections.singletonList(valueColumnMetadata)));
+ } else if (valueTimeseriesMetadataList != null) {
+ valueTimeseriesMetadataList.add(valueColumnMetadata);
}
+ } else {
+ resultTimeseriesMetadataList.add(valueColumnMetadata);
}
+ measurementFound[j] = true;
+ numOfFoundMeasurements++;
}
- return resultTimeseriesMetadataList;
+ return numOfFoundMeasurements;
}
protected int binarySearchInTimeseriesMetadataList(
@@ -669,7 +819,12 @@ public class TsFileSequenceReader implements AutoCloseable {
if (tsFileMetaData == null) {
readFileMetadata();
}
- return getAllDevices(tsFileMetaData.getMetadataIndex());
+ List<IDeviceID> deviceIDS = new ArrayList<>();
+ for (Entry<String, MetadataIndexNode> entry :
+ tsFileMetaData.getTableMetadataIndexNodeMap().entrySet()) {
+ deviceIDS.addAll(getAllDevices(entry.getValue()));
+ }
+ return deviceIDS;
}
private List<IDeviceID> getAllDevices(MetadataIndexNode metadataIndexNode) throws IOException {
@@ -690,7 +845,9 @@ public class TsFileSequenceReader implements AutoCloseable {
endOffset = metadataIndexNode.getChildren().get(i + 1).getOffset();
}
ByteBuffer buffer = readData(metadataIndexNode.getChildren().get(i).getOffset(), endOffset);
- MetadataIndexNode node = MetadataIndexNode.deserializeFrom(buffer, true);
+ MetadataIndexNode node =
+ deserializeConfig.deviceMetadataIndexNodeBufferDeserializer.deserialize(
+ buffer, deserializeConfig);
deviceList.addAll(getAllDevices(node));
}
return deviceList;
@@ -705,15 +862,16 @@ public class TsFileSequenceReader implements AutoCloseable {
readFileMetadata();
Queue<Pair<IDeviceID, long[]>> queue = new LinkedList<>();
List<long[]> leafDeviceNodeOffsets = new ArrayList<>();
- MetadataIndexNode metadataIndexNode = tsFileMetaData.getMetadataIndex();
- if (metadataIndexNode.getNodeType().equals(MetadataIndexNodeType.LEAF_DEVICE)) {
- // the first node of index tree is device leaf node, then get the devices directly
- getDevicesOfLeafNode(metadataIndexNode, queue);
- } else {
- // get all device leaf node offset
- getAllDeviceLeafNodeOffset(metadataIndexNode, leafDeviceNodeOffsets);
+ for (MetadataIndexNode metadataIndexNode :
+ tsFileMetaData.getTableMetadataIndexNodeMap().values()) {
+ if (metadataIndexNode.getNodeType().equals(MetadataIndexNodeType.LEAF_DEVICE)) {
+ // the first node of index tree is device leaf node, then get the devices directly
+ getDevicesOfLeafNode(metadataIndexNode, queue);
+ } else {
+ // get all device leaf node offset
+ getAllDeviceLeafNodeOffset(metadataIndexNode, leafDeviceNodeOffsets);
+ }
}
-
return new TsFileDeviceIterator(this, leafDeviceNodeOffsets, queue);
}
@@ -729,7 +887,9 @@ public class TsFileSequenceReader implements AutoCloseable {
throws IOException {
try {
ByteBuffer nextBuffer = readData(startOffset, endOffset);
- MetadataIndexNode deviceLeafNode = MetadataIndexNode.deserializeFrom(nextBuffer, true);
+ MetadataIndexNode deviceLeafNode =
+ deserializeConfig.deviceMetadataIndexNodeBufferDeserializer.deserialize(
+ nextBuffer, deserializeConfig);
getDevicesOfLeafNode(deviceLeafNode, measurementNodeOffsetQueue);
} catch (StopReadTsFileByInterruptException e) {
throw e;
@@ -799,7 +959,9 @@ public class TsFileSequenceReader implements AutoCloseable {
}
ByteBuffer nextBuffer = readData(startOffset, endOffset);
getAllDeviceLeafNodeOffset(
- MetadataIndexNode.deserializeFrom(nextBuffer, true), leafDeviceNodeOffsets);
+ deserializeConfig.deviceMetadataIndexNodeBufferDeserializer.deserialize(
+ nextBuffer, deserializeConfig),
+ leafDeviceNodeOffsets);
}
} catch (StopReadTsFileByInterruptException e) {
throw e;
@@ -859,17 +1021,19 @@ public class TsFileSequenceReader implements AutoCloseable {
public Iterator<List<Path>> getPathsIterator() throws IOException {
readFileMetadata();
- MetadataIndexNode metadataIndexNode = tsFileMetaData.getMetadataIndex();
- List<IMetadataIndexEntry> metadataIndexEntryList = metadataIndexNode.getChildren();
Queue<Pair<IDeviceID, Pair<Long, Long>>> queue = new LinkedList<>();
- for (int i = 0; i < metadataIndexEntryList.size(); i++) {
- IMetadataIndexEntry metadataIndexEntry = metadataIndexEntryList.get(i);
- long endOffset = metadataIndexNode.getEndOffset();
- if (i != metadataIndexEntryList.size() - 1) {
- endOffset = metadataIndexEntryList.get(i + 1).getOffset();
+ for (MetadataIndexNode metadataIndexNode :
+ tsFileMetaData.getTableMetadataIndexNodeMap().values()) {
+ List<IMetadataIndexEntry> metadataIndexEntryList = metadataIndexNode.getChildren();
+ for (int i = 0; i < metadataIndexEntryList.size(); i++) {
+ IMetadataIndexEntry metadataIndexEntry = metadataIndexEntryList.get(i);
+ long endOffset = metadataIndexNode.getEndOffset();
+ if (i != metadataIndexEntryList.size() - 1) {
+ endOffset = metadataIndexEntryList.get(i + 1).getOffset();
+ }
+ ByteBuffer buffer = readData(metadataIndexEntry.getOffset(), endOffset);
+ getAllPaths(metadataIndexEntry, buffer, null, metadataIndexNode.getNodeType(), queue);
}
- ByteBuffer buffer = readData(metadataIndexEntry.getOffset(), endOffset);
- getAllPaths(metadataIndexEntry, buffer, null, metadataIndexNode.getNodeType(), queue);
}
return new Iterator<List<Path>>() {
@Override
@@ -915,7 +1079,8 @@ public class TsFileSequenceReader implements AutoCloseable {
}
boolean currentChildLevelIsDevice = MetadataIndexNodeType.INTERNAL_DEVICE.equals(type);
MetadataIndexNode metadataIndexNode =
- MetadataIndexNode.deserializeFrom(buffer, currentChildLevelIsDevice);
+ deserializeConfig.deserializeMetadataIndexNode(buffer, currentChildLevelIsDevice);
+
int metadataIndexListSize = metadataIndexNode.getChildren().size();
for (int i = 0; i < metadataIndexListSize; i++) {
long startOffset = metadataIndexNode.getChildren().get(i).getOffset();
@@ -954,8 +1119,7 @@ public class TsFileSequenceReader implements AutoCloseable {
return "".equals(((MeasurementMetadataIndexEntry) entry).getName());
}
- TimeseriesMetadata tryToGetFirstTimeseriesMetadata(MetadataIndexNode measurementNode)
- throws IOException {
+ TimeseriesMetadata getTimeColumnMetadata(MetadataIndexNode measurementNode) throws IOException {
// Not aligned timeseries
if (!isAlignedDevice(measurementNode)) {
return null;
@@ -980,8 +1144,10 @@ public class TsFileSequenceReader implements AutoCloseable {
readData(
measurementNode.getChildren().get(0).getOffset(),
measurementNode.getChildren().get(1).getOffset());
- MetadataIndexNode metadataIndexNode = MetadataIndexNode.deserializeFrom(buffer, false);
- return tryToGetFirstTimeseriesMetadata(metadataIndexNode);
+ MetadataIndexNode metadataIndexNode =
+ deserializeConfig.measurementMetadataIndexNodeBufferDeserializer.deserialize(
+ buffer, deserializeConfig);
+ return getTimeColumnMetadata(metadataIndexNode);
}
return null;
}
@@ -1030,7 +1196,8 @@ public class TsFileSequenceReader implements AutoCloseable {
} else {
// internal measurement node
MetadataIndexNode nextLayerMeasurementNode =
- MetadataIndexNode.deserializeFrom(nextBuffer, false);
+ deserializeConfig.measurementMetadataIndexNodeBufferDeserializer.deserialize(
+ nextBuffer, deserializeConfig);
timeseriesMetadataOffsetMap.putAll(
getTimeseriesMetadataOffsetByDevice(
nextLayerMeasurementNode, excludedMeasurementIds, needChunkMetadata));
@@ -1087,7 +1254,8 @@ public class TsFileSequenceReader implements AutoCloseable {
} else {
// internal measurement node
MetadataIndexNode nextLayerMeasurementNode =
- MetadataIndexNode.deserializeFrom(nextBuffer, false);
+ deserializeConfig.measurementMetadataIndexNodeBufferDeserializer.deserialize(
+ nextBuffer, deserializeConfig);
getDeviceTimeseriesMetadata(
timeseriesMetadataList,
nextLayerMeasurementNode,
@@ -1130,7 +1298,8 @@ public class TsFileSequenceReader implements AutoCloseable {
}
boolean currentChildLevelIsDevice = MetadataIndexNodeType.INTERNAL_DEVICE.equals(type);
MetadataIndexNode metadataIndexNode =
- MetadataIndexNode.deserializeFrom(buffer, currentChildLevelIsDevice);
+ deserializeConfig.deserializeMetadataIndexNode(buffer, currentChildLevelIsDevice);
+
int metadataIndexListSize = metadataIndexNode.getChildren().size();
for (int i = 0; i < metadataIndexListSize; i++) {
long endOffset = metadataIndexNode.getEndOffset();
@@ -1196,7 +1365,7 @@ public class TsFileSequenceReader implements AutoCloseable {
}
boolean currentChildLevelIsDevice = MetadataIndexNodeType.INTERNAL_DEVICE.equals(type);
MetadataIndexNode metadataIndexNode =
- MetadataIndexNode.deserializeFrom(
+ deserializeConfig.deserializeMetadataIndexNode(
tsFileInput.wrapAsInputStream(), currentChildLevelIsDevice);
int metadataIndexListSize = metadataIndexNode.getChildren().size();
for (int i = 0; i < metadataIndexListSize; i++) {
@@ -1229,41 +1398,45 @@ public class TsFileSequenceReader implements AutoCloseable {
readFileMetadata();
}
Map<IDeviceID, List<TimeseriesMetadata>> timeseriesMetadataMap = new HashMap<>();
- MetadataIndexNode metadataIndexNode = tsFileMetaData.getMetadataIndex();
- List<IMetadataIndexEntry> metadataIndexEntryList = metadataIndexNode.getChildren();
- for (int i = 0; i < metadataIndexEntryList.size(); i++) {
- IMetadataIndexEntry metadataIndexEntry = metadataIndexEntryList.get(i);
- long endOffset = metadataIndexNode.getEndOffset();
- if (i != metadataIndexEntryList.size() - 1) {
- endOffset = metadataIndexEntryList.get(i + 1).getOffset();
- }
- if (endOffset - metadataIndexEntry.getOffset() < Integer.MAX_VALUE) {
- ByteBuffer buffer = readData(metadataIndexEntry.getOffset(), endOffset);
- generateMetadataIndex(
- metadataIndexEntry,
- buffer,
- null,
- metadataIndexNode.getNodeType(),
- timeseriesMetadataMap,
- needChunkMetadata);
- } else {
- generateMetadataIndexUsingTsFileInput(
- metadataIndexNode.getChildren().get(i),
- metadataIndexNode.getChildren().get(i).getOffset(),
- endOffset,
- null,
- metadataIndexNode.getNodeType(),
- timeseriesMetadataMap,
- needChunkMetadata);
+ for (MetadataIndexNode metadataIndexNode :
+ tsFileMetaData.getTableMetadataIndexNodeMap().values()) {
+ List<IMetadataIndexEntry> metadataIndexEntryList = metadataIndexNode.getChildren();
+ for (int i = 0; i < metadataIndexEntryList.size(); i++) {
+ IMetadataIndexEntry metadataIndexEntry = metadataIndexEntryList.get(i);
+ long endOffset = metadataIndexNode.getEndOffset();
+ if (i != metadataIndexEntryList.size() - 1) {
+ endOffset = metadataIndexEntryList.get(i + 1).getOffset();
+ }
+ if (endOffset - metadataIndexEntry.getOffset() < Integer.MAX_VALUE) {
+ ByteBuffer buffer = readData(metadataIndexEntry.getOffset(), endOffset);
+ generateMetadataIndex(
+ metadataIndexEntry,
+ buffer,
+ null,
+ metadataIndexNode.getNodeType(),
+ timeseriesMetadataMap,
+ needChunkMetadata);
+ } else {
+ generateMetadataIndexUsingTsFileInput(
+ metadataIndexNode.getChildren().get(i),
+ metadataIndexNode.getChildren().get(i).getOffset(),
+ endOffset,
+ null,
+ metadataIndexNode.getNodeType(),
+ timeseriesMetadataMap,
+ needChunkMetadata);
+ }
}
}
+
return timeseriesMetadataMap;
}
/* This method will only deserialize the TimeseriesMetadata, not including chunk metadata list */
private List<TimeseriesMetadata> getDeviceTimeseriesMetadataWithoutChunkMetadata(IDeviceID device)
throws IOException {
- MetadataIndexNode metadataIndexNode = tsFileMetaData.getMetadataIndex();
+ MetadataIndexNode metadataIndexNode =
+ tsFileMetaData.getTableMetadataIndexNode(device.getTableName());
Pair<IMetadataIndexEntry, Long> metadataIndexPair =
getMetadataAndEndOffsetOfDeviceNode(metadataIndexNode, device, true);
if (metadataIndexPair == null) {
@@ -1288,7 +1461,8 @@ public class TsFileSequenceReader implements AutoCloseable {
/* This method will not only deserialize the TimeseriesMetadata, but also all the chunk metadata list meanwhile. */
private List<TimeseriesMetadata> getDeviceTimeseriesMetadata(IDeviceID device)
throws IOException {
- MetadataIndexNode metadataIndexNode = tsFileMetaData.getMetadataIndex();
+ MetadataIndexNode metadataIndexNode =
+ tsFileMetaData.getTableMetadataIndexNode(device.getTableName());
Pair<IMetadataIndexEntry, Long> metadataIndexPair =
getMetadataAndEndOffsetOfDeviceNode(metadataIndexNode, device, true);
if (metadataIndexPair == null) {
@@ -1321,6 +1495,9 @@ public class TsFileSequenceReader implements AutoCloseable {
*/
protected Pair<IMetadataIndexEntry, Long> getMetadataAndEndOffsetOfDeviceNode(
MetadataIndexNode metadataIndex, IDeviceID deviceID, boolean exactSearch) throws IOException {
+ if (metadataIndex == null) {
+ return null;
+ }
if (MetadataIndexNodeType.INTERNAL_MEASUREMENT.equals(metadataIndex.getNodeType())
|| MetadataIndexNodeType.LEAF_MEASUREMENT.equals(metadataIndex.getNodeType())) {
throw new IllegalArgumentException();
@@ -1331,7 +1508,10 @@ public class TsFileSequenceReader implements AutoCloseable {
metadataIndex.getChildIndexEntry(deviceID, false);
ByteBuffer buffer = readData(childIndexEntry.left.getOffset(), childIndexEntry.right);
return getMetadataAndEndOffsetOfDeviceNode(
- MetadataIndexNode.deserializeFrom(buffer, true), deviceID, exactSearch);
+ deserializeConfig.deviceMetadataIndexNodeBufferDeserializer.deserialize(
+ buffer, deserializeConfig),
+ deviceID,
+ exactSearch);
} else {
return metadataIndex.getChildIndexEntry(deviceID, exactSearch);
}
@@ -1364,7 +1544,10 @@ public class TsFileSequenceReader implements AutoCloseable {
metadataIndex.getChildIndexEntry(measurement, false);
ByteBuffer buffer = readData(childIndexEntry.left.getOffset(), childIndexEntry.right);
return getMetadataAndEndOffsetOfMeasurementNode(
- MetadataIndexNode.deserializeFrom(buffer, false), measurement, exactSearch);
+ deserializeConfig.measurementMetadataIndexNodeBufferDeserializer.deserialize(
+ buffer, deserializeConfig),
+ measurement,
+ exactSearch);
} else {
return metadataIndex.getChildIndexEntry(measurement, exactSearch);
}
@@ -1528,7 +1711,6 @@ public class TsFileSequenceReader implements AutoCloseable {
*
* @param timeseriesMetadata timeseries' metadata
* @return a pair of {@link CompressionType} and {@link TSEncoding} of given timeseries.
- * @throws IOException
*/
public Pair<CompressionType, TSEncoding> readTimeseriesCompressionTypeAndEncoding(
TimeseriesMetadata timeseriesMetadata) throws IOException {
@@ -1711,9 +1893,7 @@ public class TsFileSequenceReader implements AutoCloseable {
*/
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning
public long selfCheck(
- Map<Path, IMeasurementSchema> newSchema,
- List<ChunkGroupMetadata> chunkGroupMetadataList,
- boolean fastFinish)
+ Schema schema, List<ChunkGroupMetadata> chunkGroupMetadataList, boolean fastFinish)
throws IOException {
File checkFile = FSFactoryProducer.getFSFactory().getFile(this.file);
long fileSize;
@@ -1929,7 +2109,13 @@ public class TsFileSequenceReader implements AutoCloseable {
valueColumn2TimeBatchIndex.put(chunkHeader.getMeasurementID(), timeBatchIndex + 1);
}
currentChunk =
- new ChunkMetadata(measurementID, dataType, fileOffsetOfChunk, chunkStatistics);
+ new ChunkMetadata(
+ measurementID,
+ dataType,
+ chunkHeader.getEncodingType(),
+ chunkHeader.getCompressionType(),
+ fileOffsetOfChunk,
+ chunkStatistics);
chunkMetadataList.add(currentChunk);
break;
case MetaMarker.CHUNK_GROUP_HEADER:
@@ -1938,10 +2124,9 @@ public class TsFileSequenceReader implements AutoCloseable {
truncatedSize = this.position() - 1;
if (lastDeviceId != null) {
// schema of last chunk group
- if (newSchema != null) {
+ if (schema != null) {
for (IMeasurementSchema tsSchema : measurementSchemaList) {
- newSchema.putIfAbsent(
- new Path(lastDeviceId, tsSchema.getMeasurementId(), true), tsSchema);
+ schema.registerTimeseries(lastDeviceId, tsSchema);
}
}
measurementSchemaList = new ArrayList<>();
@@ -1959,10 +2144,9 @@ public class TsFileSequenceReader implements AutoCloseable {
truncatedSize = this.position() - 1;
if (lastDeviceId != null) {
// schema of last chunk group
- if (newSchema != null) {
+ if (schema != null) {
for (IMeasurementSchema tsSchema : measurementSchemaList) {
- newSchema.putIfAbsent(
- new Path(lastDeviceId, tsSchema.getMeasurementId(), true), tsSchema);
+ schema.registerTimeseries(lastDeviceId, tsSchema);
}
}
measurementSchemaList = new ArrayList<>();
@@ -1982,10 +2166,9 @@ public class TsFileSequenceReader implements AutoCloseable {
// ChunkGroupFooter is complete.
if (lastDeviceId != null) {
// schema of last chunk group
- if (newSchema != null) {
+ if (schema != null) {
for (IMeasurementSchema tsSchema : measurementSchemaList) {
- newSchema.putIfAbsent(
- new Path(lastDeviceId, tsSchema.getMeasurementId(), true), tsSchema);
+ schema.registerTimeseries(lastDeviceId, tsSchema);
}
}
// last chunk group Metadata
@@ -2003,6 +2186,10 @@ public class TsFileSequenceReader implements AutoCloseable {
this.position(),
e.getMessage());
}
+
+ for (ChunkGroupMetadata chunkGroupMetadata : chunkGroupMetadataList) {
+ schema.updateTableSchema(chunkGroupMetadata);
+ }
// Despite the completeness of the data section, we will discard current FileMetadata
// so that we can continue to write data into this tsfile.
return truncatedSize;
@@ -2153,13 +2340,12 @@ public class TsFileSequenceReader implements AutoCloseable {
/**
* get ChunkMetaDatas of given path, and throw exception if path not exists
*
- * @param path timeseries path
* @return List of ChunkMetaData
*/
- public List<ChunkMetadata> getChunkMetadataList(Path path, boolean ignoreNotExists)
- throws IOException {
+ public List<ChunkMetadata> getChunkMetadataList(
+ IDeviceID deviceID, String measurement, boolean ignoreNotExists) throws IOException {
TimeseriesMetadata timeseriesMetaData =
- readTimeseriesMetadata(path.getIDeviceID(), path.getMeasurement(), ignoreNotExists);
+ readTimeseriesMetadata(deviceID, measurement, ignoreNotExists);
if (timeseriesMetaData == null) {
return Collections.emptyList();
}
@@ -2168,6 +2354,12 @@ public class TsFileSequenceReader implements AutoCloseable {
return chunkMetadataList;
}
+ @Deprecated
+ public List<ChunkMetadata> getChunkMetadataList(Path path, boolean ignoreNotExists)
+ throws IOException {
+ return getChunkMetadataList(path.getIDeviceID(), path.getMeasurement(), ignoreNotExists);
+ }
+
// This method is only used for TsFile
public List<IChunkMetadata> getIChunkMetadataList(Path path) throws IOException {
ITimeSeriesMetadata timeseriesMetaData = readITimeseriesMetadata(path, true);
@@ -2179,8 +2371,36 @@ public class TsFileSequenceReader implements AutoCloseable {
return chunkMetadataList;
}
+ public List<IChunkMetadata> getIChunkMetadataList(IDeviceID deviceID, String measurementName)
+ throws IOException {
+ List<ITimeSeriesMetadata> timeseriesMetaData =
+ readITimeseriesMetadata(deviceID, Collections.singleton(measurementName), null, false);
+ if (timeseriesMetaData == null || timeseriesMetaData.isEmpty()) {
+ return Collections.emptyList();
+ }
+ List<IChunkMetadata> chunkMetadataList = readIChunkMetaDataList(timeseriesMetaData.get(0));
+ chunkMetadataList.sort(Comparator.comparingLong(IChunkMetadata::getStartTime));
+ return chunkMetadataList;
+ }
+
+ public List<List<IChunkMetadata>> getIChunkMetadataList(
+ IDeviceID deviceID, Set<String> measurementNames, MetadataIndexNode root) throws IOException {
+ List<ITimeSeriesMetadata> timeseriesMetaData =
+ readITimeseriesMetadata(deviceID, measurementNames, root, true);
+ if (timeseriesMetaData == null || timeseriesMetaData.isEmpty()) {
+ return Collections.emptyList();
+ }
+ List<List<IChunkMetadata>> results = new ArrayList<>(timeseriesMetaData.size());
+ for (ITimeSeriesMetadata timeseriesMetaDatum : timeseriesMetaData) {
+ List<IChunkMetadata> chunkMetadataList = readIChunkMetaDataList(timeseriesMetaDatum);
+ chunkMetadataList.sort(Comparator.comparingLong(IChunkMetadata::getStartTime));
+ results.add(chunkMetadataList);
+ }
+ return results;
+ }
+
public List<ChunkMetadata> getChunkMetadataList(Path path) throws IOException {
- return getChunkMetadataList(path, false);
+ return getChunkMetadataList(path, true);
}
/**
@@ -2191,7 +2411,8 @@ public class TsFileSequenceReader implements AutoCloseable {
*/
public List<AlignedChunkMetadata> getAlignedChunkMetadata(IDeviceID device) throws IOException {
readFileMetadata();
- MetadataIndexNode deviceMetadataIndexNode = tsFileMetaData.getMetadataIndex();
+ MetadataIndexNode deviceMetadataIndexNode =
+ tsFileMetaData.getTableMetadataIndexNode(device.getTableName());
Pair<IMetadataIndexEntry, Long> metadataIndexPair =
getMetadataAndEndOffsetOfDeviceNode(deviceMetadataIndexNode, device, true);
if (metadataIndexPair == null) {
@@ -2201,7 +2422,9 @@ public class TsFileSequenceReader implements AutoCloseable {
MetadataIndexNode metadataIndexNode;
try {
// next layer MeasurementNode of the specific DeviceNode
- metadataIndexNode = MetadataIndexNode.deserializeFrom(buffer, false);
+ metadataIndexNode =
+ deserializeConfig.measurementMetadataIndexNodeBufferDeserializer.deserialize(
+ buffer, deserializeConfig);
} catch (Exception e) {
logger.error(METADATA_INDEX_NODE_DESERIALIZE_ERROR, file);
throw e;
@@ -2218,7 +2441,7 @@ public class TsFileSequenceReader implements AutoCloseable {
*/
public List<AlignedChunkMetadata> getAlignedChunkMetadataByMetadataIndexNode(
IDeviceID device, MetadataIndexNode metadataIndexNode) throws IOException {
- TimeseriesMetadata firstTimeseriesMetadata = tryToGetFirstTimeseriesMetadata(metadataIndexNode);
+ TimeseriesMetadata firstTimeseriesMetadata = getTimeColumnMetadata(metadataIndexNode);
if (firstTimeseriesMetadata == null) {
throw new IOException("Timeseries of device {" + device + "} are not aligned");
}
@@ -2326,7 +2549,7 @@ public class TsFileSequenceReader implements AutoCloseable {
Map<String, TimeseriesMetadata> timeseriesMetadataMap = readDeviceMetadata(device);
for (TimeseriesMetadata timeseriesMetadata : timeseriesMetadataMap.values()) {
result.put(
- ((PlainDeviceID) device).toStringID()
+ device.toString()
+ TsFileConstant.PATH_SEPARATOR
+ timeseriesMetadata.getMeasurementId(),
timeseriesMetadata.getTsDataType());
@@ -2376,7 +2599,18 @@ public class TsFileSequenceReader implements AutoCloseable {
*/
public MetadataIndexNode readMetadataIndexNode(
long startOffset, long endOffset, boolean isDeviceLevel) throws IOException {
- return MetadataIndexNode.deserializeFrom(readData(startOffset, endOffset), isDeviceLevel);
+ MetadataIndexNode metadataIndexNode;
+ final ByteBuffer buffer = readData(startOffset, endOffset);
+ if (isDeviceLevel) {
+ metadataIndexNode =
+ deserializeConfig.deviceMetadataIndexNodeBufferDeserializer.deserialize(
+ buffer, deserializeConfig);
+ } else {
+ metadataIndexNode =
+ deserializeConfig.measurementMetadataIndexNodeBufferDeserializer.deserialize(
+ buffer, deserializeConfig);
+ }
+ return metadataIndexNode;
}
/**
@@ -2433,7 +2667,8 @@ public class TsFileSequenceReader implements AutoCloseable {
IDeviceID device) throws IOException {
readFileMetadata();
- MetadataIndexNode metadataIndexNode = tsFileMetaData.getMetadataIndex();
+ MetadataIndexNode metadataIndexNode =
+ tsFileMetaData.getTableMetadataIndexNode(device.getTableName());
Pair<IMetadataIndexEntry, Long> metadataIndexPair =
getMetadataAndEndOffsetOfDeviceNode(metadataIndexNode, device, true);
@@ -2453,7 +2688,10 @@ public class TsFileSequenceReader implements AutoCloseable {
}
ByteBuffer buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right);
- MetadataIndexNode firstMeasurementNode = MetadataIndexNode.deserializeFrom(buffer, false);
+
+ MetadataIndexNode firstMeasurementNode =
+ deserializeConfig.measurementMetadataIndexNodeBufferDeserializer.deserialize(
+ buffer, deserializeConfig);
return getMeasurementChunkMetadataListMapIterator(firstMeasurementNode);
}
@@ -2523,7 +2761,9 @@ public class TsFileSequenceReader implements AutoCloseable {
continue;
}
collectEachLeafMeasurementNodeOffsetRange(
- MetadataIndexNode.deserializeFrom(readData(startOffset, endOffset), false), queue);
+ deserializeConfig.measurementMetadataIndexNodeBufferDeserializer.deserialize(
+ readData(startOffset, endOffset), deserializeConfig),
+ queue);
}
} catch (StopReadTsFileByInterruptException e) {
throw e;
@@ -2550,4 +2790,8 @@ public class TsFileSequenceReader implements AutoCloseable {
public int hashCode() {
return Objects.hash(file);
}
+
+ public DeserializeConfig getDeserializeContext() {
+ return deserializeConfig;
+ }
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReaderTimeseriesMetadataIterator.java b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReaderTimeseriesMetadataIterator.java
index 2fc83d95..9fa1c311 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReaderTimeseriesMetadataIterator.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReaderTimeseriesMetadataIterator.java
@@ -19,6 +19,7 @@
package org.apache.tsfile.read;
+import org.apache.tsfile.compatibility.DeserializeConfig;
import org.apache.tsfile.exception.TsFileSequenceReaderTimeseriesMetadataIteratorException;
import org.apache.tsfile.file.IMetadataIndexEntry;
import org.apache.tsfile.file.metadata.DeviceMetadataIndexEntry;
@@ -50,6 +51,7 @@ public class TsFileSequenceReaderTimeseriesMetadataIterator
private final Deque<MetadataIndexEntryInfo> metadataIndexEntryStack = new ArrayDeque<>();
private IDeviceID currentDeviceId;
private int currentTimeseriesMetadataCount = 0;
+ private DeserializeConfig deserializeConfig;
public TsFileSequenceReaderTimeseriesMetadataIterator(
TsFileSequenceReader reader, boolean needChunkMetadata, int timeseriesBatchReadNumber)
@@ -57,20 +59,23 @@ public class TsFileSequenceReaderTimeseriesMetadataIterator
this.reader = reader;
this.needChunkMetadata = needChunkMetadata;
this.timeseriesBatchReadNumber = timeseriesBatchReadNumber;
+ this.deserializeConfig = new DeserializeConfig();
if (this.reader.tsFileMetaData == null) {
this.reader.readFileMetadata();
}
- final MetadataIndexNode metadataIndexNode = reader.tsFileMetaData.getMetadataIndex();
- long curEntryEndOffset = metadataIndexNode.getEndOffset();
- List<IMetadataIndexEntry> metadataIndexEntryList = metadataIndexNode.getChildren();
+ for (MetadataIndexNode metadataIndexNode :
+ reader.tsFileMetaData.getTableMetadataIndexNodeMap().values()) {
+ long curEntryEndOffset = metadataIndexNode.getEndOffset();
+ List<IMetadataIndexEntry> metadataIndexEntryList = metadataIndexNode.getChildren();
- for (int i = metadataIndexEntryList.size() - 1; i >= 0; i--) {
- metadataIndexEntryStack.push(
- new MetadataIndexEntryInfo(
- metadataIndexEntryList.get(i), metadataIndexNode.getNodeType(), curEntryEndOffset));
- curEntryEndOffset = metadataIndexEntryList.get(i).getOffset();
+ for (int i = metadataIndexEntryList.size() - 1; i >= 0; i--) {
+ metadataIndexEntryStack.push(
+ new MetadataIndexEntryInfo(
+ metadataIndexEntryList.get(i), metadataIndexNode.getNodeType(), curEntryEndOffset));
+ curEntryEndOffset = metadataIndexEntryList.get(i).getOffset();
+ }
}
}
@@ -222,7 +227,7 @@ public class TsFileSequenceReaderTimeseriesMetadataIterator
boolean currentChildLevelIsDevice = MetadataIndexNodeType.INTERNAL_DEVICE.equals(type);
final MetadataIndexNode metadataIndexNode =
- MetadataIndexNode.deserializeFrom(
+ deserializeConfig.deserializeMetadataIndexNode(
reader.readData(metadataIndexEntry.getOffset(), endOffset), currentChildLevelIsDevice);
MetadataIndexNodeType metadataIndexNodeType = metadataIndexNode.getNodeType();
List<IMetadataIndexEntry> children = metadataIndexNode.getChildren();
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/Path.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/Path.java
index 3a682bf1..52968495 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/Path.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/Path.java
@@ -22,9 +22,9 @@ package org.apache.tsfile.read.common;
import org.apache.tsfile.common.constant.TsFileConstant;
import org.apache.tsfile.exception.PathParseException;
import org.apache.tsfile.file.metadata.IDeviceID;
-import org.apache.tsfile.file.metadata.PlainDeviceID;
+import org.apache.tsfile.file.metadata.IDeviceID.Deserializer;
+import org.apache.tsfile.file.metadata.IDeviceID.Factory;
import org.apache.tsfile.read.common.parser.PathNodesGenerator;
-import org.apache.tsfile.utils.PublicBAOS;
import org.apache.tsfile.utils.ReadWriteIOUtils;
import org.apache.commons.lang3.StringUtils;
@@ -46,7 +46,7 @@ public class Path implements Serializable, Comparable<Path> {
private static final long serialVersionUID = 3405277066329298200L;
private String measurement;
- protected String device;
+ protected IDeviceID device;
protected String fullPath;
private static final String ILLEGAL_PATH_ARGUMENT = "Path parameter is null";
@@ -54,7 +54,8 @@ public class Path implements Serializable, Comparable<Path> {
// Only used for test
public Path(IDeviceID deviceID) {
- this(((PlainDeviceID) deviceID).toStringID());
+ this.device = deviceID;
+ this.fullPath = deviceID.toString();
}
/**
@@ -79,25 +80,29 @@ public class Path implements Serializable, Comparable<Path> {
if (!needSplit) {
// no split, we don't use antlr to check here.
fullPath = pathSc;
+ device = Factory.DEFAULT_FACTORY.create(pathSc);
} else {
if (pathSc.length() > 0) {
String[] nodes = PathNodesGenerator.splitPathToNodes(pathSc);
- device = "";
if (nodes.length > 1) {
- device = transformNodesToString(nodes, nodes.length - 1);
+ device = Factory.DEFAULT_FACTORY.create(transformNodesToString(nodes, nodes.length - 1));
+ } else {
+ device = Factory.DEFAULT_FACTORY.create("");
}
measurement = nodes[nodes.length - 1];
- fullPath = transformNodesToString(nodes, nodes.length);
+ fullPath = this.device.toString() + TsFileConstant.PATH_SEPARATOR + measurement;
} else {
fullPath = pathSc;
- device = "";
+ device = Factory.DEFAULT_FACTORY.create("");
measurement = pathSc;
}
}
}
public Path(IDeviceID device, String measurement, boolean needCheck) {
- this(((PlainDeviceID) device).toStringID(), measurement, needCheck);
+ this.device = device;
+ this.measurement = measurement;
+ this.fullPath = this.device.toString() + TsFileConstant.PATH_SEPARATOR + measurement;
}
/**
@@ -113,7 +118,7 @@ public class Path implements Serializable, Comparable<Path> {
}
if (!needCheck) {
this.measurement = measurement;
- this.device = device;
+ this.device = Factory.DEFAULT_FACTORY.create(device);
this.fullPath = device + TsFileConstant.PATH_SEPARATOR + measurement;
return;
}
@@ -121,22 +126,23 @@ public class Path implements Serializable, Comparable<Path> {
if (!StringUtils.isEmpty(device) && !StringUtils.isEmpty(measurement)) {
String path = device + TsFileConstant.PATH_SEPARATOR + measurement;
String[] nodes = PathNodesGenerator.splitPathToNodes(path);
- this.device = transformNodesToString(nodes, nodes.length - 1);
+ this.device = Factory.DEFAULT_FACTORY.create(transformNodesToString(nodes, nodes.length - 1));
this.measurement = nodes[nodes.length - 1];
- this.fullPath = transformNodesToString(nodes, nodes.length);
+ this.fullPath = this.device.toString() + TsFileConstant.PATH_SEPARATOR + measurement;
} else if (!StringUtils.isEmpty(device)) {
String[] deviceNodes = PathNodesGenerator.splitPathToNodes(device);
- this.device = transformNodesToString(deviceNodes, deviceNodes.length);
+ this.device =
+ Factory.DEFAULT_FACTORY.create(transformNodesToString(deviceNodes, deviceNodes.length));
this.measurement = measurement;
// for aligned path, sensor name for time column is ""
this.fullPath = device + TsFileConstant.PATH_SEPARATOR + measurement;
} else if (!StringUtils.isEmpty(measurement)) {
String[] measurementNodes = PathNodesGenerator.splitPathToNodes(measurement);
this.measurement = transformNodesToString(measurementNodes, measurementNodes.length);
- this.device = device;
+ this.device = Factory.DEFAULT_FACTORY.create(device);
this.fullPath = measurement;
} else {
- this.device = device;
+ this.device = Factory.DEFAULT_FACTORY.create(device);
this.measurement = measurement;
this.fullPath = "";
}
@@ -146,12 +152,12 @@ public class Path implements Serializable, Comparable<Path> {
return fullPath;
}
- public String getDevice() {
- return device;
+ public String getDeviceString() {
+ return device.toString();
}
public IDeviceID getIDeviceID() {
- return new PlainDeviceID(getDevice());
+ return device;
}
public String getMeasurement() {
@@ -210,11 +216,6 @@ public class Path implements Serializable, Comparable<Path> {
serializeWithoutType(stream);
}
- public void serialize(PublicBAOS stream) throws IOException {
- ReadWriteIOUtils.write((byte) 3, stream); // org.apache.iotdb.db.metadata.path#PathType
- serializeWithoutType(stream);
- }
-
protected void serializeWithoutType(ByteBuffer byteBuffer) {
if (measurement == null) {
ReadWriteIOUtils.write((byte) 0, byteBuffer);
@@ -226,7 +227,7 @@ public class Path implements Serializable, Comparable<Path> {
ReadWriteIOUtils.write((byte) 0, byteBuffer);
} else {
ReadWriteIOUtils.write((byte) 1, byteBuffer);
- ReadWriteIOUtils.write(device, byteBuffer);
+ device.serialize(byteBuffer);
}
if (fullPath == null) {
ReadWriteIOUtils.write((byte) 0, byteBuffer);
@@ -247,28 +248,7 @@ public class Path implements Serializable, Comparable<Path> {
ReadWriteIOUtils.write((byte) 0, stream);
} else {
ReadWriteIOUtils.write((byte) 1, stream);
- ReadWriteIOUtils.write(device, stream);
- }
- if (fullPath == null) {
- ReadWriteIOUtils.write((byte) 0, stream);
- } else {
- ReadWriteIOUtils.write((byte) 1, stream);
- ReadWriteIOUtils.write(fullPath, stream);
- }
- }
-
- protected void serializeWithoutType(PublicBAOS stream) throws IOException {
- if (measurement == null) {
- ReadWriteIOUtils.write((byte) 0, stream);
- } else {
- ReadWriteIOUtils.write((byte) 1, stream);
- ReadWriteIOUtils.write(measurement, stream);
- }
- if (device == null) {
- ReadWriteIOUtils.write((byte) 0, stream);
- } else {
- ReadWriteIOUtils.write((byte) 1, stream);
- ReadWriteIOUtils.write(device, stream);
+ device.serialize(stream);
}
if (fullPath == null) {
ReadWriteIOUtils.write((byte) 0, stream);
@@ -283,7 +263,8 @@ public class Path implements Serializable, Comparable<Path> {
byte isNull = ReadWriteIOUtils.readByte(byteBuffer);
path.measurement = isNull == 0 ? null : ReadWriteIOUtils.readString(byteBuffer);
isNull = ReadWriteIOUtils.readByte(byteBuffer);
- path.device = isNull == 0 ? null : ReadWriteIOUtils.readString(byteBuffer);
+ path.device =
+ isNull == 0 ? null : Deserializer.DEFAULT_DESERIALIZER.deserializeFrom(byteBuffer);
isNull = ReadWriteIOUtils.readByte(byteBuffer);
path.fullPath = isNull == 0 ? null : ReadWriteIOUtils.readString(byteBuffer);
return path;
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlock.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlock.java
index 14072b7a..c607e906 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlock.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlock.java
@@ -20,16 +20,20 @@
package org.apache.tsfile.read.common.block;
import org.apache.tsfile.block.column.Column;
+import org.apache.tsfile.file.metadata.TableSchema;
import org.apache.tsfile.read.TimeValuePair;
import org.apache.tsfile.read.common.IBatchDataIterator;
+import org.apache.tsfile.read.common.block.column.ColumnFactory;
import org.apache.tsfile.read.common.block.column.TimeColumn;
import org.apache.tsfile.read.reader.IPointReader;
import org.apache.tsfile.utils.RamUsageEstimator;
import org.apache.tsfile.utils.TsPrimitiveType;
import org.apache.tsfile.write.UnSupportedDataTypeException;
+import org.apache.tsfile.write.schema.IMeasurementSchema;
import java.util.Arrays;
import java.util.Iterator;
+import java.util.List;
import java.util.NoSuchElementException;
import static java.lang.String.format;
@@ -52,16 +56,16 @@ public class TsBlock {
* doesn't defensively copy the valueColumns
*/
public static TsBlock wrapBlocksWithoutCopy(
- int positionCount, TimeColumn timeColumn, Column[] valueColumns) {
+ int positionCount, Column timeColumn, Column[] valueColumns) {
return new TsBlock(false, positionCount, timeColumn, valueColumns);
}
- private final TimeColumn timeColumn;
+ private final Column timeColumn;
private final Column[] valueColumns;
/** How many rows in current TsBlock */
- private final int positionCount;
+ private int positionCount;
private volatile long retainedSizeInBytes = -1;
@@ -69,19 +73,16 @@ public class TsBlock {
this(false, positionCount, null, EMPTY_COLUMNS);
}
- public TsBlock(TimeColumn timeColumn, Column... valueColumns) {
+ public TsBlock(Column timeColumn, Column... valueColumns) {
this(true, determinePositionCount(valueColumns), timeColumn, valueColumns);
}
- public TsBlock(int positionCount, TimeColumn timeColumn, Column... valueColumns) {
+ public TsBlock(int positionCount, Column timeColumn, Column... valueColumns) {
this(true, positionCount, timeColumn, valueColumns);
}
private TsBlock(
- boolean columnsCopyRequired,
- int positionCount,
- TimeColumn timeColumn,
- Column[] valueColumns) {
+ boolean columnsCopyRequired, int positionCount, Column timeColumn, Column[] valueColumns) {
requireNonNull(valueColumns, "blocks is null");
this.positionCount = positionCount;
this.timeColumn = timeColumn;
@@ -98,12 +99,16 @@ public class TsBlock {
return positionCount;
}
+ public void setPositionCount(int positionCount) {
+ this.positionCount = positionCount;
+ }
+
public long getStartTime() {
- return timeColumn.getStartTime();
+ return timeColumn.getLong(0);
}
public long getEndTime() {
- return timeColumn.getEndTime();
+ return timeColumn.getLong(positionCount - 1);
}
public boolean isEmpty() {
@@ -135,7 +140,7 @@ public class TsBlock {
slicedColumns[i] = valueColumns[i].getRegion(positionOffset, length);
}
return wrapBlocksWithoutCopy(
- length, (TimeColumn) timeColumn.getRegion(positionOffset, length), slicedColumns);
+ length, timeColumn.getRegion(positionOffset, length), slicedColumns);
}
/**
@@ -166,7 +171,7 @@ public class TsBlock {
if (fromIndex > positionCount) {
throw new IllegalArgumentException("FromIndex of subTsBlock cannot over positionCount.");
}
- TimeColumn subTimeColumn = (TimeColumn) timeColumn.subColumn(fromIndex);
+ Column subTimeColumn = timeColumn.subColumn(fromIndex);
Column[] subValueColumns = new Column[valueColumns.length];
for (int i = 0; i < subValueColumns.length; i++) {
subValueColumns[i] = valueColumns[i].subColumn(fromIndex);
@@ -186,7 +191,7 @@ public class TsBlock {
return valueColumns.length;
}
- public TimeColumn getTimeColumn() {
+ public Column getTimeColumn() {
return timeColumn;
}
@@ -248,6 +253,17 @@ public class TsBlock {
return new TsBlockAlignedRowIterator(0);
}
+ public void reset() {
+ if (positionCount == 0) {
+ return;
+ }
+ positionCount = 0;
+ timeColumn.reset();
+ for (Column valueColumn : valueColumns) {
+ valueColumn.reset();
+ }
+ }
+
public class TsBlockSingleColumnIterator implements IPointReader, IBatchDataIterator {
private int rowIndex;
@@ -523,7 +539,11 @@ public class TsBlock {
}
public void update(int updateIdx, TsBlock sourceTsBlock, int sourceIndex) {
- timeColumn.getTimes()[updateIdx] = sourceTsBlock.getTimeByIndex(sourceIndex);
+ timeColumn.getLongs()[updateIdx] = sourceTsBlock.getTimeByIndex(sourceIndex);
+ updateWithoutTimeColumn(updateIdx, sourceTsBlock, sourceIndex);
+ }
+
+ public void updateWithoutTimeColumn(int updateIdx, TsBlock sourceTsBlock, int sourceIndex) {
for (int i = 0; i < getValueColumnCount(); i++) {
if (sourceTsBlock.getValueColumns()[i].isNull(sourceIndex)) {
valueColumns[i].isNull()[updateIdx] = true;
@@ -570,4 +590,27 @@ public class TsBlock {
}
}
}
+
+ public static TsBlock buildTsBlock(List<String> columnNames, TableSchema schema, int blockSize) {
+ Column timeColumn = new TimeColumn(blockSize);
+ Column[] columns = new Column[columnNames.size()];
+ for (int i = 0; i < columnNames.size(); i++) {
+ final String columnName = columnNames.get(i);
+ final IMeasurementSchema columnSchema = schema.findColumnSchema(columnName);
+ columns[i] = ColumnFactory.create(columnSchema.getType(), blockSize);
+ }
+ return new TsBlock(timeColumn, columns);
+ }
+
+ /**
+ * For each column, if its positionCount < this. positionCount, add nulls at the end of the
+ * column.
+ */
+ public void fillTrailingNulls() {
+ for (Column valueColumn : valueColumns) {
+ if (valueColumn.getPositionCount() < this.positionCount) {
+ valueColumn.setNull(valueColumn.getPositionCount(), this.positionCount);
+ }
+ }
+ }
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlockBuilder.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlockBuilder.java
index 568c222c..918551d9 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlockBuilder.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlockBuilder.java
@@ -310,6 +310,10 @@ public class TsBlockBuilder {
public TsBlock build() {
TimeColumn timeColumn = (TimeColumn) timeColumnBuilder.build();
+ return build(timeColumn);
+ }
+
+ public TsBlock build(Column timeColumn) {
if (timeColumn.getPositionCount() != declaredPositions) {
throw new IllegalStateException(
format(
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlockUtil.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlockUtil.java
index f0dd1551..106d568e 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlockUtil.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlockUtil.java
@@ -19,8 +19,8 @@
package org.apache.tsfile.read.common.block;
+import org.apache.tsfile.block.column.Column;
import org.apache.tsfile.read.common.TimeRange;
-import org.apache.tsfile.read.common.block.column.TimeColumn;
import org.apache.tsfile.read.filter.basic.Filter;
import org.apache.tsfile.read.reader.series.PaginationController;
@@ -41,7 +41,7 @@ public class TsBlockUtil {
// else, find the index of first less than or equal to targetTime
public static int getFirstConditionIndex(
TsBlock tsBlock, TimeRange targetTimeRange, boolean ascending) {
- TimeColumn timeColumn = tsBlock.getTimeColumn();
+ Column timeColumn = tsBlock.getTimeColumn();
long targetTime = ascending ? targetTimeRange.getMin() : targetTimeRange.getMax();
int left = 0;
int right = timeColumn.getPositionCount() - 1;
@@ -49,19 +49,19 @@ public class TsBlockUtil {
while (left < right) {
mid = (left + right) >> 1;
- if (timeColumn.getLongWithoutCheck(mid) < targetTime) {
+ if (timeColumn.getLong(mid) < targetTime) {
if (ascending) {
left = mid + 1;
} else {
right = mid;
}
- } else if (timeColumn.getLongWithoutCheck(mid) > targetTime) {
+ } else if (timeColumn.getLong(mid) > targetTime) {
if (ascending) {
right = mid;
} else {
left = mid + 1;
}
- } else if (timeColumn.getLongWithoutCheck(mid) == targetTime) {
+ } else if (timeColumn.getLong(mid) == targetTime) {
return mid;
}
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumn.java
index beb48c28..0be88efe 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumn.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumn.java
@@ -39,12 +39,16 @@ public class BinaryColumn implements Column {
(int) RamUsageEstimator.shallowSizeOfInstance(BinaryColumn.class);
private final int arrayOffset;
- private final int positionCount;
- private final boolean[] valueIsNull;
+ private int positionCount;
+ private boolean[] valueIsNull;
private final Binary[] values;
private final long retainedSizeInBytes;
+ public BinaryColumn(int initialCapacity) {
+ this(0, 0, null, new Binary[initialCapacity]);
+ }
+
public BinaryColumn(int positionCount, Optional<boolean[]> valueIsNull, Binary[] values) {
this(0, positionCount, valueIsNull.orElse(null), values);
}
@@ -197,4 +201,17 @@ public class BinaryColumn implements Column {
public int getInstanceSize() {
return INSTANCE_SIZE;
}
+
+ @Override
+ public void setPositionCount(int count) {
+ positionCount = count;
+ }
+
+ @Override
+ public void setNull(int start, int end) {
+ if (valueIsNull == null) {
+ valueIsNull = new boolean[values.length];
+ }
+ Arrays.fill(valueIsNull, start, end, true);
+ }
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BooleanColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BooleanColumn.java
index f74bcd8e..d534a845 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BooleanColumn.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BooleanColumn.java
@@ -38,12 +38,16 @@ public class BooleanColumn implements Column {
public static final int SIZE_IN_BYTES_PER_POSITION = Byte.BYTES + Byte.BYTES;
private final int arrayOffset;
- private final int positionCount;
- private final boolean[] valueIsNull;
+ private int positionCount;
+ private boolean[] valueIsNull;
private final boolean[] values;
private final long retainedSizeInBytes;
+ public BooleanColumn(int initialCapacity) {
+ this(0, 0, null, new boolean[initialCapacity]);
+ }
+
public BooleanColumn(int positionCount, Optional<boolean[]> valueIsNull, boolean[] values) {
this(0, positionCount, valueIsNull.orElse(null), values);
}
@@ -195,4 +199,17 @@ public class BooleanColumn implements Column {
public int getInstanceSize() {
return INSTANCE_SIZE;
}
+
+ @Override
+ public void setPositionCount(int count) {
+ positionCount = count;
+ }
+
+ @Override
+ public void setNull(int start, int end) {
+ if (valueIsNull == null) {
+ valueIsNull = new boolean[values.length];
+ }
+ Arrays.fill(valueIsNull, start, end, true);
+ }
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/ColumnEncoder.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/ColumnEncoder.java
index b65841b8..9f8965e4 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/ColumnEncoder.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/ColumnEncoder.java
@@ -28,11 +28,6 @@ import java.nio.ByteBuffer;
public interface ColumnEncoder {
- /** Read a time column from the specified input. */
- default TimeColumn readTimeColumn(ByteBuffer input, int positionCount) {
- throw new UnsupportedOperationException();
- }
-
/** Read a column from the specified input. */
Column readColumn(ByteBuffer input, TSDataType dataType, int positionCount);
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeFactory.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/ColumnFactory.java
similarity index 60%
copy from java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeFactory.java
copy to java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/ColumnFactory.java
index 72946ee8..42984c3d 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeFactory.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/ColumnFactory.java
@@ -17,37 +17,28 @@
* under the License.
*/
-package org.apache.tsfile.read.common.type;
+package org.apache.tsfile.read.common.block.column;
+import org.apache.tsfile.block.column.Column;
import org.apache.tsfile.enums.TSDataType;
-public class TypeFactory {
-
- private TypeFactory() {
- // forbidding instantiation
- }
-
- public static Type getType(TSDataType tsDataType) {
- switch (tsDataType) {
- case INT32:
- case DATE:
- return IntType.getInstance();
+public class ColumnFactory {
+ public static Column create(TSDataType dataType, int initialCapacity) {
+ switch (dataType) {
case INT64:
- case TIMESTAMP:
- return LongType.getInstance();
- case FLOAT:
- return FloatType.getInstance();
+ return new LongColumn(initialCapacity);
case DOUBLE:
- return DoubleType.getInstance();
- case BOOLEAN:
- return BooleanType.getInstance();
+ return new DoubleColumn(initialCapacity);
+ case FLOAT:
+ return new FloatColumn(initialCapacity);
case TEXT:
- case BLOB:
- case STRING:
- return BinaryType.getInstance();
+ return new BinaryColumn(initialCapacity);
+ case INT32:
+ return new IntColumn(initialCapacity);
+ case BOOLEAN:
+ return new BooleanColumn(initialCapacity);
default:
- throw new UnsupportedOperationException(
- String.format("Invalid TSDataType for TypeFactory: %s", tsDataType));
+ throw new IllegalArgumentException("Unsupported data type: " + dataType);
}
}
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DoubleColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DoubleColumn.java
index 0311830b..c065d98d 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DoubleColumn.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DoubleColumn.java
@@ -39,12 +39,16 @@ public class DoubleColumn implements Column {
public static final int SIZE_IN_BYTES_PER_POSITION = Double.BYTES + Byte.BYTES;
private final int arrayOffset;
- private final int positionCount;
- private final boolean[] valueIsNull;
+ private int positionCount;
+ private boolean[] valueIsNull;
private final double[] values;
private final long retainedSizeInBytes;
+ public DoubleColumn(int initialCapacity) {
+ this(0, 0, null, new double[initialCapacity]);
+ }
+
public DoubleColumn(int positionCount, Optional<boolean[]> valueIsNull, double[] values) {
this(0, positionCount, valueIsNull.orElse(null), values);
}
@@ -196,4 +200,17 @@ public class DoubleColumn implements Column {
public int getInstanceSize() {
return INSTANCE_SIZE;
}
+
+ @Override
+ public void setPositionCount(int count) {
+ positionCount = count;
+ }
+
+ @Override
+ public void setNull(int start, int end) {
+ if (valueIsNull == null) {
+ valueIsNull = new boolean[values.length];
+ }
+ Arrays.fill(valueIsNull, start, end, true);
+ }
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/FloatColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/FloatColumn.java
index 449212eb..56306736 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/FloatColumn.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/FloatColumn.java
@@ -39,12 +39,16 @@ public class FloatColumn implements Column {
public static final int SIZE_IN_BYTES_PER_POSITION = Float.BYTES + Byte.BYTES;
private final int arrayOffset;
- private final int positionCount;
- private final boolean[] valueIsNull;
+ private int positionCount;
+ private boolean[] valueIsNull;
private final float[] values;
private final long retainedSizeInBytes;
+ public FloatColumn(int initialCapacity) {
+ this(0, 0, null, new float[initialCapacity]);
+ }
+
public FloatColumn(int positionCount, Optional<boolean[]> valueIsNull, float[] values) {
this(0, positionCount, valueIsNull.orElse(null), values);
}
@@ -195,4 +199,17 @@ public class FloatColumn implements Column {
public int getInstanceSize() {
return INSTANCE_SIZE;
}
+
+ @Override
+ public void setPositionCount(int count) {
+ positionCount = count;
+ }
+
+ @Override
+ public void setNull(int start, int end) {
+ if (valueIsNull == null) {
+ valueIsNull = new boolean[values.length];
+ }
+ Arrays.fill(valueIsNull, start, end, true);
+ }
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/Int64ArrayColumnEncoder.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/Int64ArrayColumnEncoder.java
index 58cfbebf..446adaf4 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/Int64ArrayColumnEncoder.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/Int64ArrayColumnEncoder.java
@@ -28,28 +28,6 @@ import java.nio.ByteBuffer;
public class Int64ArrayColumnEncoder implements ColumnEncoder {
- @Override
- public TimeColumn readTimeColumn(ByteBuffer input, int positionCount) {
-
- // Serialized data layout:
- // +---------------+-----------------+-------------+
- // | may have null | null indicators | values |
- // +---------------+-----------------+-------------+
- // | byte | list[byte] | list[int64] |
- // +---------------+-----------------+-------------+
-
- boolean[] nullIndicators = ColumnEncoder.deserializeNullIndicators(input, positionCount);
- long[] values = new long[positionCount];
- if (nullIndicators == null) {
- for (int i = 0; i < positionCount; i++) {
- values[i] = input.getLong();
- }
- return new TimeColumn(0, positionCount, values);
- } else {
- throw new IllegalArgumentException("TimeColumn should not contain null values.");
- }
- }
-
@Override
public Column readColumn(ByteBuffer input, TSDataType dataType, int positionCount) {
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumn.java
index 0254690b..2c3c35c1 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumn.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumn.java
@@ -39,12 +39,16 @@ public class IntColumn implements Column {
public static final int SIZE_IN_BYTES_PER_POSITION = Integer.BYTES + Byte.BYTES;
private final int arrayOffset;
- private final int positionCount;
- private final boolean[] valueIsNull;
+ private int positionCount;
+ private boolean[] valueIsNull;
private final int[] values;
private final long retainedSizeInBytes;
+ public IntColumn(int initialCapacity) {
+ this(0, 0, null, new int[initialCapacity]);
+ }
+
public IntColumn(int positionCount, Optional<boolean[]> valueIsNull, int[] values) {
this(0, positionCount, valueIsNull.orElse(null), values);
}
@@ -195,4 +199,17 @@ public class IntColumn implements Column {
public int getInstanceSize() {
return INSTANCE_SIZE;
}
+
+ @Override
+ public void setPositionCount(int count) {
+ this.positionCount = count;
+ }
+
+ @Override
+ public void setNull(int start, int end) {
+ if (valueIsNull == null) {
+ valueIsNull = new boolean[values.length];
+ }
+ Arrays.fill(valueIsNull, start, end, true);
+ }
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/LongColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/LongColumn.java
index 34cc7dc4..1c022369 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/LongColumn.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/LongColumn.java
@@ -39,12 +39,16 @@ public class LongColumn implements Column {
public static final int SIZE_IN_BYTES_PER_POSITION = Long.BYTES + Byte.BYTES;
private final int arrayOffset;
- private final int positionCount;
- private final boolean[] valueIsNull;
+ private int positionCount;
+ private boolean[] valueIsNull;
private final long[] values;
private final long retainedSizeInBytes;
+ public LongColumn(int initialCapacity) {
+ this(0, 0, null, new long[initialCapacity]);
+ }
+
public LongColumn(int positionCount, Optional<boolean[]> valueIsNull, long[] values) {
this(0, positionCount, valueIsNull.orElse(null), values);
}
@@ -195,4 +199,17 @@ public class LongColumn implements Column {
public int getInstanceSize() {
return INSTANCE_SIZE;
}
+
+ @Override
+ public void setPositionCount(int count) {
+ this.positionCount = count;
+ }
+
+ @Override
+ public void setNull(int start, int end) {
+ if (valueIsNull == null) {
+ valueIsNull = new boolean[values.length];
+ }
+ Arrays.fill(valueIsNull, start, end, true);
+ }
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/NullColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/NullColumn.java
index 8b7e2152..9b2a7bcc 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/NullColumn.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/NullColumn.java
@@ -36,7 +36,7 @@ public class NullColumn implements Column {
private static final int INSTANCE_SIZE =
(int) RamUsageEstimator.shallowSizeOfInstance(BooleanColumn.class);
- private final int positionCount;
+ private int positionCount;
private final long retainedSizeInBytes;
@@ -140,4 +140,12 @@ public class NullColumn implements Column {
public int getInstanceSize() {
return INSTANCE_SIZE;
}
+
+ @Override
+ public void setPositionCount(int count) {
+ this.positionCount = count;
+ }
+
+ @Override
+ public void setNull(int start, int end) {}
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/RunLengthEncodedColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/RunLengthEncodedColumn.java
index a78e2daa..d20cc443 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/RunLengthEncodedColumn.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/RunLengthEncodedColumn.java
@@ -37,7 +37,7 @@ public class RunLengthEncodedColumn implements Column {
(int) RamUsageEstimator.shallowSizeOfInstance(RunLengthEncodedColumn.class);
private final Column value;
- private final int positionCount;
+ private int positionCount;
public RunLengthEncodedColumn(Column value, int positionCount) {
requireNonNull(value, "value is null");
@@ -230,4 +230,14 @@ public class RunLengthEncodedColumn implements Column {
public int getInstanceSize() {
return INSTANCE_SIZE;
}
+
+ @Override
+ public void setPositionCount(int count) {
+ this.positionCount = count;
+ }
+
+ @Override
+ public void setNull(int start, int end) {
+ value.setNull(start, end);
+ }
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/TimeColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/TimeColumn.java
index 1783f6a3..d681b079 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/TimeColumn.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/TimeColumn.java
@@ -31,16 +31,20 @@ import static org.apache.tsfile.utils.RamUsageEstimator.sizeOfLongArray;
public class TimeColumn implements Column {
private static final int INSTANCE_SIZE =
- (int) RamUsageEstimator.shallowSizeOfInstance(LongColumn.class);
+ (int) RamUsageEstimator.shallowSizeOfInstance(TimeColumn.class);
public static final int SIZE_IN_BYTES_PER_POSITION = Long.BYTES;
private final int arrayOffset;
- private final int positionCount;
+ private int positionCount;
private final long[] values;
private final long retainedSizeInBytes;
+ public TimeColumn(int initialCapacity) {
+ this(0, 0, new long[initialCapacity]);
+ }
+
public TimeColumn(int positionCount, long[] values) {
this(0, positionCount, values);
}
@@ -78,10 +82,6 @@ public class TimeColumn implements Column {
return values[position + arrayOffset];
}
- public long getLongWithoutCheck(int position) {
- return values[position + arrayOffset];
- }
-
@Override
public Object getObject(int position) {
return getLong(position);
@@ -180,4 +180,12 @@ public class TimeColumn implements Column {
public int getInstanceSize() {
return INSTANCE_SIZE;
}
+
+ @Override
+ public void setPositionCount(int count) {
+ this.positionCount = positionCount;
+ }
+
+ @Override
+ public void setNull(int start, int end) {}
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/TsBlockSerde.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/TsBlockSerde.java
index 0ef00256..e887a577 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/TsBlockSerde.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/TsBlockSerde.java
@@ -67,8 +67,10 @@ public class TsBlockSerde {
}
// Time column.
- TimeColumn timeColumn =
- ColumnEncoderFactory.get(columnEncodings.get(0)).readTimeColumn(byteBuffer, positionCount);
+ // TODO: a TimeColumn will be deserialized as a LongColumn
+ Column timeColumn =
+ ColumnEncoderFactory.get(columnEncodings.get(0))
+ .readColumn(byteBuffer, TSDataType.INT64, positionCount);
// Value columns
Column[] valueColumns = new Column[valueColumnCount];
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/parser/PathNodesGenerator.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/parser/PathNodesGenerator.java
index 1d2d14ae..7c54598a 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/parser/PathNodesGenerator.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/parser/PathNodesGenerator.java
@@ -41,6 +41,9 @@ public class PathNodesGenerator {
public static String[] splitPathToNodes(String path) throws PathParseException {
try {
+ if (path.isEmpty()) {
+ return new String[] {path};
+ }
return invokeParser(path);
} catch (ParseCancellationException e) {
throw new PathParseException(path);
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/IntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java
similarity index 86%
copy from java/tsfile/src/main/java/org/apache/tsfile/read/common/type/IntType.java
copy to java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java
index 1cead88d..8425a879 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/IntType.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java
@@ -23,11 +23,10 @@ import org.apache.tsfile.block.column.Column;
import org.apache.tsfile.block.column.ColumnBuilder;
import org.apache.tsfile.read.common.block.column.IntColumnBuilder;
-public class IntType implements Type {
+import java.util.Collections;
+import java.util.List;
- private static final IntType INSTANCE = new IntType();
-
- private IntType() {}
+public abstract class AbstractIntType extends AbstractType {
@Override
public int getInt(Column c, int position) {
@@ -75,11 +74,17 @@ public class IntType implements Type {
}
@Override
- public TypeEnum getTypeEnum() {
- return TypeEnum.INT32;
+ public boolean isComparable() {
+ return true;
}
- public static IntType getInstance() {
- return INSTANCE;
+ @Override
+ public boolean isOrderable() {
+ return true;
+ }
+
+ @Override
+ public List<Type> getTypeParameters() {
+ return Collections.emptyList();
}
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/LongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java
similarity index 86%
copy from java/tsfile/src/main/java/org/apache/tsfile/read/common/type/LongType.java
copy to java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java
index 7be7b792..e8e1ecbc 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/LongType.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java
@@ -23,11 +23,10 @@ import org.apache.tsfile.block.column.Column;
import org.apache.tsfile.block.column.ColumnBuilder;
import org.apache.tsfile.read.common.block.column.LongColumnBuilder;
-public class LongType implements Type {
+import java.util.Collections;
+import java.util.List;
- private static final LongType INSTANCE = new LongType();
-
- private LongType() {}
+public abstract class AbstractLongType extends AbstractType {
@Override
public int getInt(Column c, int position) {
@@ -75,11 +74,17 @@ public class LongType implements Type {
}
@Override
- public TypeEnum getTypeEnum() {
- return TypeEnum.INT64;
+ public boolean isComparable() {
+ return true;
}
- public static LongType getInstance() {
- return INSTANCE;
+ @Override
+ public boolean isOrderable() {
+ return true;
+ }
+
+ @Override
+ public List<Type> getTypeParameters() {
+ return Collections.emptyList();
}
}
diff --git a/cpp/examples/c_examples/c_examples.h b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java
similarity index 82%
copy from cpp/examples/c_examples/c_examples.h
copy to java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java
index d29090cb..937b7fdc 100644
--- a/cpp/examples/c_examples/c_examples.h
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java
@@ -17,16 +17,12 @@
* under the License.
*/
+package org.apache.tsfile.read.common.type;
-#include "cwrapper/TsFile-cwrapper.h"
+public abstract class AbstractType implements Type {
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-ErrorCode write_tsfile();
-ErrorCode read_tsfile();
-
-#ifdef __cplusplus
+ @Override
+ public String toString() {
+ return getDisplayName();
+ }
}
-#endif
\ No newline at end of file
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BinaryType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java
similarity index 81%
copy from java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BinaryType.java
copy to java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java
index bcbd2a0b..d09ed07c 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BinaryType.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java
@@ -24,11 +24,10 @@ import org.apache.tsfile.block.column.ColumnBuilder;
import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder;
import org.apache.tsfile.utils.Binary;
-public class BinaryType implements Type {
- private static final BinaryType INSTANCE = new BinaryType();
-
- private BinaryType() {}
+import java.util.Collections;
+import java.util.List;
+public abstract class AbstractVarcharType extends AbstractType {
@Override
public Binary getBinary(Column c, int position) {
return c.getBinary(position);
@@ -45,11 +44,17 @@ public class BinaryType implements Type {
}
@Override
- public TypeEnum getTypeEnum() {
- return TypeEnum.BINARY;
+ public boolean isComparable() {
+ return true;
}
- public static BinaryType getInstance() {
- return INSTANCE;
+ @Override
+ public boolean isOrderable() {
+ return true;
+ }
+
+ @Override
+ public List<Type> getTypeParameters() {
+ return Collections.emptyList();
}
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BinaryType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BinaryType.java
index bcbd2a0b..68c87499 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BinaryType.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BinaryType.java
@@ -19,37 +19,22 @@
package org.apache.tsfile.read.common.type;
-import org.apache.tsfile.block.column.Column;
-import org.apache.tsfile.block.column.ColumnBuilder;
-import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder;
-import org.apache.tsfile.utils.Binary;
-
-public class BinaryType implements Type {
- private static final BinaryType INSTANCE = new BinaryType();
+public class BinaryType extends AbstractVarcharType {
+ public static final BinaryType TEXT = new BinaryType();
private BinaryType() {}
@Override
- public Binary getBinary(Column c, int position) {
- return c.getBinary(position);
- }
-
- @Override
- public void writeBinary(ColumnBuilder builder, Binary value) {
- builder.writeBinary(value);
- }
-
- @Override
- public ColumnBuilder createColumnBuilder(int expectedEntries) {
- return new BinaryColumnBuilder(null, expectedEntries);
+ public TypeEnum getTypeEnum() {
+ return TypeEnum.TEXT;
}
@Override
- public TypeEnum getTypeEnum() {
- return TypeEnum.BINARY;
+ public String getDisplayName() {
+ return "TEXT";
}
public static BinaryType getInstance() {
- return INSTANCE;
+ return TEXT;
}
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BinaryType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java
similarity index 72%
copy from java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BinaryType.java
copy to java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java
index bcbd2a0b..d02ca716 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BinaryType.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java
@@ -24,10 +24,14 @@ import org.apache.tsfile.block.column.ColumnBuilder;
import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder;
import org.apache.tsfile.utils.Binary;
-public class BinaryType implements Type {
- private static final BinaryType INSTANCE = new BinaryType();
+import java.util.Collections;
+import java.util.List;
- private BinaryType() {}
+public class BlobType extends AbstractType {
+
+ public static final BlobType BLOB = new BlobType();
+
+ private BlobType() {}
@Override
public Binary getBinary(Column c, int position) {
@@ -46,10 +50,30 @@ public class BinaryType implements Type {
@Override
public TypeEnum getTypeEnum() {
- return TypeEnum.BINARY;
+ return TypeEnum.BLOB;
+ }
+
+ @Override
+ public String getDisplayName() {
+ return "BLOB";
+ }
+
+ @Override
+ public boolean isComparable() {
+ return true;
+ }
+
+ @Override
+ public boolean isOrderable() {
+ return true;
+ }
+
+ @Override
+ public List<Type> getTypeParameters() {
+ return Collections.emptyList();
}
- public static BinaryType getInstance() {
- return INSTANCE;
+ public static BlobType getInstance() {
+ return BLOB;
}
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java
index 67cf86cc..e00d7ecb 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java
@@ -23,9 +23,12 @@ import org.apache.tsfile.block.column.Column;
import org.apache.tsfile.block.column.ColumnBuilder;
import org.apache.tsfile.read.common.block.column.BooleanColumnBuilder;
-public class BooleanType implements Type {
+import java.util.Collections;
+import java.util.List;
- private static final BooleanType INSTANCE = new BooleanType();
+public class BooleanType extends AbstractType {
+
+ public static final BooleanType BOOLEAN = new BooleanType();
private BooleanType() {}
@@ -49,7 +52,27 @@ public class BooleanType implements Type {
return TypeEnum.BOOLEAN;
}
+ @Override
+ public String getDisplayName() {
+ return "BOOLEAN";
+ }
+
+ @Override
+ public boolean isComparable() {
+ return true;
+ }
+
+ @Override
+ public boolean isOrderable() {
+ return true;
+ }
+
+ @Override
+ public List<Type> getTypeParameters() {
+ return Collections.emptyList();
+ }
+
public static BooleanType getInstance() {
- return INSTANCE;
+ return BOOLEAN;
}
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeEnum.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java
similarity index 72%
copy from java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeEnum.java
copy to java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java
index 5a4489de..88d17e24 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeEnum.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java
@@ -19,16 +19,23 @@
package org.apache.tsfile.read.common.type;
-public enum TypeEnum {
- INT32,
+public class DateType extends AbstractIntType {
- INT64,
+ public static final DateType DATE = new DateType();
- FLOAT,
+ private DateType() {}
- DOUBLE,
+ @Override
+ public TypeEnum getTypeEnum() {
+ return TypeEnum.DATE;
+ }
- BOOLEAN,
+ @Override
+ public String getDisplayName() {
+ return "DATE";
+ }
- BINARY
+ public static DateType getInstance() {
+ return DATE;
+ }
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java
index 317dfd96..6e325c06 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java
@@ -23,9 +23,12 @@ import org.apache.tsfile.block.column.Column;
import org.apache.tsfile.block.column.ColumnBuilder;
import org.apache.tsfile.read.common.block.column.DoubleColumnBuilder;
-public class DoubleType implements Type {
+import java.util.Collections;
+import java.util.List;
- private static final DoubleType INSTANCE = new DoubleType();
+public class DoubleType extends AbstractType {
+
+ public static final DoubleType DOUBLE = new DoubleType();
private DoubleType() {}
@@ -79,7 +82,27 @@ public class DoubleType implements Type {
return TypeEnum.DOUBLE;
}
+ @Override
+ public String getDisplayName() {
+ return "DOUBLE";
+ }
+
+ @Override
+ public boolean isComparable() {
+ return true;
+ }
+
+ @Override
+ public boolean isOrderable() {
+ return true;
+ }
+
+ @Override
+ public List<Type> getTypeParameters() {
+ return Collections.emptyList();
+ }
+
public static DoubleType getInstance() {
- return INSTANCE;
+ return DOUBLE;
}
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java
index 51e7e5b2..6417f0f5 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java
@@ -23,9 +23,12 @@ import org.apache.tsfile.block.column.Column;
import org.apache.tsfile.block.column.ColumnBuilder;
import org.apache.tsfile.read.common.block.column.FloatColumnBuilder;
-public class FloatType implements Type {
+import java.util.Collections;
+import java.util.List;
- private static final FloatType INSTANCE = new FloatType();
+public class FloatType extends AbstractType {
+
+ public static final FloatType FLOAT = new FloatType();
private FloatType() {}
@@ -79,7 +82,27 @@ public class FloatType implements Type {
return TypeEnum.FLOAT;
}
+ @Override
+ public String getDisplayName() {
+ return "FLOAT";
+ }
+
+ @Override
+ public boolean isComparable() {
+ return true;
+ }
+
+ @Override
+ public boolean isOrderable() {
+ return true;
+ }
+
+ @Override
+ public List<Type> getTypeParameters() {
+ return Collections.emptyList();
+ }
+
public static FloatType getInstance() {
- return INSTANCE;
+ return FLOAT;
}
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/IntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/IntType.java
index 1cead88d..e4d36c2a 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/IntType.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/IntType.java
@@ -19,67 +19,23 @@
package org.apache.tsfile.read.common.type;
-import org.apache.tsfile.block.column.Column;
-import org.apache.tsfile.block.column.ColumnBuilder;
-import org.apache.tsfile.read.common.block.column.IntColumnBuilder;
+public class IntType extends AbstractIntType {
-public class IntType implements Type {
-
- private static final IntType INSTANCE = new IntType();
+ public static final IntType INT32 = new IntType();
private IntType() {}
@Override
- public int getInt(Column c, int position) {
- return c.getInt(position);
- }
-
- @Override
- public long getLong(Column c, int position) {
- return c.getInt(position);
- }
-
- @Override
- public float getFloat(Column c, int position) {
- return c.getInt(position);
- }
-
- @Override
- public double getDouble(Column c, int position) {
- return c.getInt(position);
- }
-
- @Override
- public void writeInt(ColumnBuilder builder, int value) {
- builder.writeInt(value);
- }
-
- @Override
- public void writeLong(ColumnBuilder builder, long value) {
- builder.writeInt((int) value);
- }
-
- @Override
- public void writeFloat(ColumnBuilder builder, float value) {
- builder.writeInt((int) value);
- }
-
- @Override
- public void writeDouble(ColumnBuilder builder, double value) {
- builder.writeInt((int) value);
- }
-
- @Override
- public ColumnBuilder createColumnBuilder(int expectedEntries) {
- return new IntColumnBuilder(null, expectedEntries);
+ public TypeEnum getTypeEnum() {
+ return TypeEnum.INT32;
}
@Override
- public TypeEnum getTypeEnum() {
- return TypeEnum.INT32;
+ public String getDisplayName() {
+ return "INT32";
}
public static IntType getInstance() {
- return INSTANCE;
+ return INT32;
}
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/LongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/LongType.java
index 7be7b792..a1ed2a1c 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/LongType.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/LongType.java
@@ -19,67 +19,23 @@
package org.apache.tsfile.read.common.type;
-import org.apache.tsfile.block.column.Column;
-import org.apache.tsfile.block.column.ColumnBuilder;
-import org.apache.tsfile.read.common.block.column.LongColumnBuilder;
+public class LongType extends AbstractLongType {
-public class LongType implements Type {
-
- private static final LongType INSTANCE = new LongType();
+ public static final LongType INT64 = new LongType();
private LongType() {}
@Override
- public int getInt(Column c, int position) {
- return (int) c.getLong(position);
- }
-
- @Override
- public long getLong(Column c, int position) {
- return c.getLong(position);
- }
-
- @Override
- public float getFloat(Column c, int position) {
- return c.getLong(position);
- }
-
- @Override
- public double getDouble(Column c, int position) {
- return c.getLong(position);
- }
-
- @Override
- public void writeInt(ColumnBuilder builder, int value) {
- builder.writeLong(value);
- }
-
- @Override
- public void writeLong(ColumnBuilder builder, long value) {
- builder.writeLong(value);
- }
-
- @Override
- public void writeFloat(ColumnBuilder builder, float value) {
- builder.writeLong((long) value);
- }
-
- @Override
- public void writeDouble(ColumnBuilder builder, double value) {
- builder.writeLong((long) value);
- }
-
- @Override
- public ColumnBuilder createColumnBuilder(int expectedEntries) {
- return new LongColumnBuilder(null, expectedEntries);
+ public TypeEnum getTypeEnum() {
+ return TypeEnum.INT64;
}
@Override
- public TypeEnum getTypeEnum() {
- return TypeEnum.INT64;
+ public String getDisplayName() {
+ return "INT64";
}
public static LongType getInstance() {
- return INSTANCE;
+ return INT64;
}
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/RowType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/RowType.java
new file mode 100644
index 00000000..0c5dabbc
--- /dev/null
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/RowType.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tsfile.read.common.type;
+
+import org.apache.tsfile.block.column.ColumnBuilder;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+import static java.util.Objects.requireNonNull;
+import static org.apache.tsfile.read.common.type.TypeEnum.ROW;
+
+public class RowType extends AbstractType {
+
+ private final List<Field> fields;
+ private final List<Type> fieldTypes;
+ private final boolean comparable;
+ private final boolean orderable;
+
+ private RowType(List<Field> originalFields) {
+
+ this.fields = new ArrayList<>(originalFields);
+ this.fieldTypes = fields.stream().map(Field::getType).collect(Collectors.toList());
+
+ this.comparable = fields.stream().allMatch(field -> field.getType().isComparable());
+ this.orderable = fields.stream().allMatch(field -> field.getType().isOrderable());
+ }
+
+ public static RowType from(List<Field> fields) {
+ return new RowType(fields);
+ }
+
+ public static RowType anonymous(List<Type> types) {
+ List<Field> fields =
+ types.stream().map(type -> new Field(Optional.empty(), type)).collect(Collectors.toList());
+
+ return new RowType(fields);
+ }
+
+ public static RowType rowType(Field... field) {
+ return from(Arrays.asList(field));
+ }
+
+ public static RowType anonymousRow(Type... types) {
+ return anonymous(Arrays.asList(types));
+ }
+
+ // Only RowParametricType.createType should call this method
+ public static RowType createWithTypeSignature(List<Field> fields) {
+ return new RowType(fields);
+ }
+
+ public static Field field(String name, Type type) {
+ return new Field(Optional.of(name), type);
+ }
+
+ public static Field field(Type type) {
+ return new Field(Optional.empty(), type);
+ }
+
+ @Override
+ public ColumnBuilder createColumnBuilder(int expectedEntries) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public TypeEnum getTypeEnum() {
+ return ROW;
+ }
+
+ @Override
+ public String getDisplayName() {
+ // Convert to standard sql name
+ StringBuilder result = new StringBuilder();
+ result.append("ROW").append('(');
+ for (Field field : fields) {
+ String typeDisplayName = field.getType().getDisplayName();
+ if (field.getName().isPresent()) {
+ // TODO: names are already canonicalized, so they should be printed as delimited identifiers
+ result.append(field.getName().get()).append(' ').append(typeDisplayName);
+ } else {
+ result.append(typeDisplayName);
+ }
+ result.append(", ");
+ }
+ result.setLength(result.length() - 2);
+ result.append(')');
+ return result.toString();
+ }
+
+ @Override
+ public List<Type> getTypeParameters() {
+ return fieldTypes;
+ }
+
+ public List<Field> getFields() {
+ return fields;
+ }
+
+ public static class Field {
+ private final Type type;
+ private final Optional<String> name;
+
+ public Field(Optional<String> name, Type type) {
+ this.type = requireNonNull(type, "type is null");
+ this.name = requireNonNull(name, "name is null");
+ }
+
+ public Type getType() {
+ return type;
+ }
+
+ public Optional<String> getName() {
+ return name;
+ }
+ }
+
+ @Override
+ public boolean isComparable() {
+ return comparable;
+ }
+
+ @Override
+ public boolean isOrderable() {
+ return orderable;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ RowType rowType = (RowType) o;
+ return comparable == rowType.comparable
+ && orderable == rowType.orderable
+ && Objects.equals(fields, rowType.fields)
+ && Objects.equals(fieldTypes, rowType.fieldTypes);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(fields, fieldTypes, comparable, orderable);
+ }
+}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeEnum.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/StringType.java
similarity index 70%
copy from java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeEnum.java
copy to java/tsfile/src/main/java/org/apache/tsfile/read/common/type/StringType.java
index 5a4489de..725ae9e3 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeEnum.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/StringType.java
@@ -19,16 +19,23 @@
package org.apache.tsfile.read.common.type;
-public enum TypeEnum {
- INT32,
+public class StringType extends AbstractVarcharType {
- INT64,
+ public static final StringType STRING = new StringType();
- FLOAT,
+ private StringType() {}
- DOUBLE,
+ @Override
+ public TypeEnum getTypeEnum() {
+ return TypeEnum.STRING;
+ }
- BOOLEAN,
+ @Override
+ public String getDisplayName() {
+ return "STRING";
+ }
- BINARY
+ public static StringType getInstance() {
+ return STRING;
+ }
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeEnum.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TimestampType.java
similarity index 69%
copy from java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeEnum.java
copy to java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TimestampType.java
index 5a4489de..c4e5e843 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeEnum.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TimestampType.java
@@ -19,16 +19,23 @@
package org.apache.tsfile.read.common.type;
-public enum TypeEnum {
- INT32,
+public class TimestampType extends AbstractLongType {
- INT64,
+ public static final TimestampType TIMESTAMP = new TimestampType();
- FLOAT,
+ private TimestampType() {}
- DOUBLE,
+ @Override
+ public TypeEnum getTypeEnum() {
+ return TypeEnum.TIMESTAMP;
+ }
- BOOLEAN,
+ @Override
+ public String getDisplayName() {
+ return "TIMESTAMP";
+ }
- BINARY
+ public static TimestampType getInstance() {
+ return TIMESTAMP;
+ }
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java
index 118b83e7..cb6b6094 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java
@@ -23,6 +23,8 @@ import org.apache.tsfile.block.column.Column;
import org.apache.tsfile.block.column.ColumnBuilder;
import org.apache.tsfile.utils.Binary;
+import java.util.List;
+
public interface Type {
/** Gets a boolean at {@code position}. */
@@ -102,4 +104,16 @@ public interface Type {
ColumnBuilder createColumnBuilder(int expectedEntries);
TypeEnum getTypeEnum();
+
+ /** Returns the name of this type that should be displayed to end-users. */
+ String getDisplayName();
+
+ /** True if the type supports equalTo and hash. */
+ boolean isComparable();
+
+ /** True if the type supports compareTo. */
+ boolean isOrderable();
+
+ /** For parameterized types returns the list of parameters. */
+ List<Type> getTypeParameters();
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeEnum.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeEnum.java
index 5a4489de..184a64af 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeEnum.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeEnum.java
@@ -30,5 +30,13 @@ public enum TypeEnum {
BOOLEAN,
- BINARY
+ TEXT,
+
+ ROW,
+
+ UNKNOWN,
+ TIMESTAMP,
+ DATE,
+ BLOB,
+ STRING
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeFactory.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeFactory.java
index 72946ee8..41474542 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeFactory.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeFactory.java
@@ -29,11 +29,37 @@ public class TypeFactory {
public static Type getType(TSDataType tsDataType) {
switch (tsDataType) {
- case INT32:
case DATE:
+ return DateType.getInstance();
+ case INT32:
return IntType.getInstance();
case INT64:
+ return LongType.getInstance();
case TIMESTAMP:
+ return TimestampType.getInstance();
+ case FLOAT:
+ return FloatType.getInstance();
+ case DOUBLE:
+ return DoubleType.getInstance();
+ case BOOLEAN:
+ return BooleanType.getInstance();
+ case TEXT:
+ return BinaryType.getInstance();
+ case STRING:
+ return StringType.getInstance();
+ case BLOB:
+ return BlobType.getInstance();
+ default:
+ throw new UnsupportedOperationException(
+ String.format("Invalid TSDataType for TypeFactory: %s", tsDataType));
+ }
+ }
+
+ public static Type getType(TypeEnum typeEnum) {
+ switch (typeEnum) {
+ case INT32:
+ return IntType.getInstance();
+ case INT64:
return LongType.getInstance();
case FLOAT:
return FloatType.getInstance();
@@ -42,12 +68,20 @@ public class TypeFactory {
case BOOLEAN:
return BooleanType.getInstance();
case TEXT:
+ return BinaryType.getInstance();
+ case UNKNOWN:
+ return UnknownType.getInstance();
+ case DATE:
+ return DateType.getInstance();
+ case TIMESTAMP:
+ return TimestampType.getInstance();
case BLOB:
+ return BlobType.getInstance();
case STRING:
- return BinaryType.getInstance();
+ return StringType.getInstance();
default:
throw new UnsupportedOperationException(
- String.format("Invalid TSDataType for TypeFactory: %s", tsDataType));
+ String.format("Invalid TypeEnum for TypeFactory: %s", typeEnum));
}
}
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/UnknownType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/UnknownType.java
new file mode 100644
index 00000000..818019cf
--- /dev/null
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/UnknownType.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tsfile.read.common.type;
+
+import org.apache.tsfile.block.column.Column;
+import org.apache.tsfile.block.column.ColumnBuilder;
+import org.apache.tsfile.read.common.block.column.BooleanColumnBuilder;
+
+import java.util.Collections;
+import java.util.List;
+
+import static org.apache.tsfile.utils.Preconditions.checkArgument;
+
+public class UnknownType implements Type {
+ public static final UnknownType UNKNOWN = new UnknownType();
+ public static final String NAME = "unknown";
+
+ private UnknownType() {}
+
+ @Override
+ public void writeBoolean(ColumnBuilder columnBuilder, boolean value) {
+ // Ideally, this function should never be invoked for the unknown type.
+ // However, some logic (e.g. AbstractMinMaxBy) relies on writing a default value before the null
+ // check.
+ checkArgument(!value);
+ columnBuilder.appendNull();
+ }
+
+ @Override
+ public boolean getBoolean(Column column, int position) {
+ // Ideally, this function should never be invoked for the unknown type.
+ // However, some logic relies on having a default value before the null check.
+ checkArgument(column.isNull(position));
+ return false;
+ }
+
+ @Override
+ public ColumnBuilder createColumnBuilder(int expectedEntries) {
+ return new BooleanColumnBuilder(null, expectedEntries);
+ }
+
+ @Override
+ public TypeEnum getTypeEnum() {
+ return TypeEnum.UNKNOWN;
+ }
+
+ @Override
+ public String getDisplayName() {
+ return NAME;
+ }
+
+ @Override
+ public boolean isComparable() {
+ return true;
+ }
+
+ @Override
+ public boolean isOrderable() {
+ return true;
+ }
+
+ @Override
+ public List<Type> getTypeParameters() {
+ return Collections.emptyList();
+ }
+
+ public static UnknownType getInstance() {
+ return UNKNOWN;
+ }
+}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/controller/DeviceMetaIterator.java b/java/tsfile/src/main/java/org/apache/tsfile/read/controller/DeviceMetaIterator.java
new file mode 100644
index 00000000..4445bf9b
--- /dev/null
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/controller/DeviceMetaIterator.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tsfile.read.controller;
+
+import org.apache.tsfile.file.IMetadataIndexEntry;
+import org.apache.tsfile.file.metadata.DeviceMetadataIndexEntry;
+import org.apache.tsfile.file.metadata.IDeviceID;
+import org.apache.tsfile.file.metadata.MetadataIndexNode;
+import org.apache.tsfile.file.metadata.enums.MetadataIndexNodeType;
+import org.apache.tsfile.read.TsFileSequenceReader;
+import org.apache.tsfile.read.expression.ExpressionTree;
+import org.apache.tsfile.utils.Pair;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayDeque;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.Queue;
+
+public class DeviceMetaIterator implements Iterator<Pair<IDeviceID, MetadataIndexNode>> {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(DeviceMetaIterator.class);
+ private final TsFileSequenceReader tsFileSequenceReader;
+ private final Queue<MetadataIndexNode> metadataIndexNodes = new ArrayDeque<>();
+ private final Queue<Pair<IDeviceID, MetadataIndexNode>> resultCache = new ArrayDeque<>();
+ private final ExpressionTree idFilter;
+
+ public DeviceMetaIterator(
+ TsFileSequenceReader tsFileSequenceReader,
+ MetadataIndexNode metadataIndexNode,
+ ExpressionTree idFilter) {
+ this.tsFileSequenceReader = tsFileSequenceReader;
+ this.metadataIndexNodes.add(metadataIndexNode);
+ this.idFilter = idFilter;
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (!resultCache.isEmpty()) {
+ return true;
+ }
+ try {
+ loadResults();
+ } catch (IOException e) {
+ LOGGER.error("Failed to load device meta data", e);
+ return false;
+ }
+
+ return !resultCache.isEmpty();
+ }
+
+ private void loadLeafDevice(MetadataIndexNode currentNode) throws IOException {
+ List<IMetadataIndexEntry> leafChildren = currentNode.getChildren();
+ for (int i = 0; i < leafChildren.size(); i++) {
+ IMetadataIndexEntry child = leafChildren.get(i);
+ final IDeviceID deviceID = ((DeviceMetadataIndexEntry) child).getDeviceID();
+ if (idFilter != null && !idFilter.satisfy(deviceID)) {
+ continue;
+ }
+
+ long startOffset = child.getOffset();
+ long endOffset =
+ i < leafChildren.size() - 1
+ ? leafChildren.get(i + 1).getOffset()
+ : currentNode.getEndOffset();
+ final MetadataIndexNode childNode =
+ tsFileSequenceReader.readMetadataIndexNode(startOffset, endOffset, false);
+ resultCache.add(new Pair<>(deviceID, childNode));
+ }
+ }
+
+ private void loadInternalNode(MetadataIndexNode currentNode) throws IOException {
+ List<IMetadataIndexEntry> internalChildren = currentNode.getChildren();
+ for (int i = 0; i < internalChildren.size(); i++) {
+ IMetadataIndexEntry child = internalChildren.get(i);
+ long startOffset = child.getOffset();
+ long endOffset =
+ i < internalChildren.size() - 1
+ ? internalChildren.get(i + 1).getOffset()
+ : currentNode.getEndOffset();
+ final MetadataIndexNode childNode =
+ tsFileSequenceReader.readMetadataIndexNode(startOffset, endOffset, true);
+ metadataIndexNodes.add(childNode);
+ }
+ }
+
+ private void loadResults() throws IOException {
+ while (!metadataIndexNodes.isEmpty()) {
+ final MetadataIndexNode currentNode = metadataIndexNodes.poll();
+ final MetadataIndexNodeType nodeType = currentNode.getNodeType();
+ switch (nodeType) {
+ case LEAF_DEVICE:
+ loadLeafDevice(currentNode);
+ if (!resultCache.isEmpty()) {
+ return;
+ }
+ case INTERNAL_DEVICE:
+ loadInternalNode(currentNode);
+ break;
+ default:
+ throw new IOException("A non-device node detected: " + currentNode);
+ }
+ }
+ }
+
+ @Override
+ public Pair<IDeviceID, MetadataIndexNode> next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ return resultCache.poll();
+ }
+}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/controller/IMetadataQuerier.java b/java/tsfile/src/main/java/org/apache/tsfile/read/controller/IMetadataQuerier.java
index 3aa63a6b..b6d9654a 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/controller/IMetadataQuerier.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/controller/IMetadataQuerier.java
@@ -22,18 +22,35 @@ package org.apache.tsfile.read.controller;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.exception.write.NoMeasurementException;
import org.apache.tsfile.file.metadata.IChunkMetadata;
+import org.apache.tsfile.file.metadata.IDeviceID;
+import org.apache.tsfile.file.metadata.MetadataIndexNode;
import org.apache.tsfile.file.metadata.TsFileMetadata;
import org.apache.tsfile.read.common.Path;
import org.apache.tsfile.read.common.TimeRange;
+import org.apache.tsfile.read.expression.ExpressionTree;
+import org.apache.tsfile.utils.Pair;
import java.io.IOException;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Set;
public interface IMetadataQuerier {
List<IChunkMetadata> getChunkMetaDataList(Path path) throws IOException;
+ /**
+ * @param deviceID the deviceID to be queried
+ * @param measurementNames the measurementNames to be queried
+ * @param measurementNode nullable, if provided, the search will start from the node
+ * @return each list is the ChunkMetadata of those timeseries who exists
+ * @throws IOException if IO error occurs
+ */
+ List<List<IChunkMetadata>> getChunkMetadataLists(
+ IDeviceID deviceID, Set<String> measurementNames, MetadataIndexNode measurementNode)
+ throws IOException;
+
Map<Path, List<IChunkMetadata>> getChunkMetaDataMap(List<Path> paths) throws IOException;
TsFileMetadata getWholeFileMetadata();
@@ -65,4 +82,7 @@ public interface IMetadataQuerier {
/** clear caches (if used) to release memory. */
void clear();
+
+ Iterator<Pair<IDeviceID, MetadataIndexNode>> deviceIterator(
+ MetadataIndexNode root, ExpressionTree idFilter);
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/controller/MetadataQuerierByFileImpl.java b/java/tsfile/src/main/java/org/apache/tsfile/read/controller/MetadataQuerierByFileImpl.java
index 64d62442..00e3ae1c 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/controller/MetadataQuerierByFileImpl.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/controller/MetadataQuerierByFileImpl.java
@@ -26,12 +26,15 @@ import org.apache.tsfile.file.metadata.ChunkMetadata;
import org.apache.tsfile.file.metadata.IChunkMetadata;
import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.file.metadata.ITimeSeriesMetadata;
+import org.apache.tsfile.file.metadata.MetadataIndexNode;
import org.apache.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.tsfile.file.metadata.TsFileMetadata;
import org.apache.tsfile.read.TsFileSequenceReader;
import org.apache.tsfile.read.TsFileSequenceReader.LocateStatus;
import org.apache.tsfile.read.common.Path;
import org.apache.tsfile.read.common.TimeRange;
+import org.apache.tsfile.read.expression.ExpressionTree;
+import org.apache.tsfile.utils.Pair;
import java.io.IOException;
import java.util.ArrayList;
@@ -39,6 +42,7 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -52,8 +56,8 @@ public class MetadataQuerierByFileImpl implements IMetadataQuerier {
private TsFileMetadata fileMetaData;
- // TimeseriesPath -> List<IChunkMetadata>
- private LRUCache<Path, List<IChunkMetadata>> chunkMetaDataCache;
+ // (deviceId, measurementId) -> List<IChunkMetadata>
+ private LRUCache<Pair<IDeviceID, String>, List<IChunkMetadata>> deviceIdChunkMetadataCache;
private TsFileSequenceReader tsFileReader;
@@ -61,10 +65,11 @@ public class MetadataQuerierByFileImpl implements IMetadataQuerier {
public MetadataQuerierByFileImpl(TsFileSequenceReader tsFileReader) throws IOException {
this.tsFileReader = tsFileReader;
this.fileMetaData = tsFileReader.readFileMetadata();
- chunkMetaDataCache =
- new LRUCache<Path, List<IChunkMetadata>>(CACHED_ENTRY_NUMBER) {
+ deviceIdChunkMetadataCache =
+ new LRUCache<Pair<IDeviceID, String>, List<IChunkMetadata>>(CACHED_ENTRY_NUMBER) {
@Override
- public List<IChunkMetadata> loadObjectByKey(Path key) throws IOException {
+ protected List<IChunkMetadata> loadObjectByKey(Pair<IDeviceID, String> key)
+ throws IOException {
return loadChunkMetadata(key);
}
};
@@ -72,7 +77,37 @@ public class MetadataQuerierByFileImpl implements IMetadataQuerier {
@Override
public List<IChunkMetadata> getChunkMetaDataList(Path timeseriesPath) throws IOException {
- return new ArrayList<>(chunkMetaDataCache.get(timeseriesPath));
+ return new ArrayList<>(
+ deviceIdChunkMetadataCache.get(
+ new Pair<>(timeseriesPath.getIDeviceID(), timeseriesPath.getMeasurement())));
+ }
+
+ public List<List<IChunkMetadata>> getChunkMetadataLists(
+ IDeviceID deviceID, Set<String> measurementNames, MetadataIndexNode measurementNode)
+ throws IOException {
+ List<List<IChunkMetadata>> results = new ArrayList<>(measurementNames.size());
+ final Iterator<String> iterator = measurementNames.iterator();
+ // use cache when possible
+ while (iterator.hasNext()) {
+ final String measurementName = iterator.next();
+ // check first to avoid loading
+ final Pair<IDeviceID, String> key = new Pair<>(deviceID, measurementName);
+ if (deviceIdChunkMetadataCache.containsKey(key)) {
+ final List<IChunkMetadata> metadataList = deviceIdChunkMetadataCache.get(key);
+ results.add(metadataList);
+ iterator.remove();
+ }
+ }
+ // the remaining is not in the cache, search them in file
+ final List<List<IChunkMetadata>> iChunkMetadataList =
+ tsFileReader.getIChunkMetadataList(deviceID, measurementNames, measurementNode);
+ for (List<IChunkMetadata> metadataList : iChunkMetadataList) {
+ final String measurementUid = metadataList.get(0).getMeasurementUid();
+ // cache the result
+ deviceIdChunkMetadataCache.put(new Pair<>(deviceID, measurementUid), metadataList);
+ results.add(metadataList);
+ }
+ return results;
}
@Override
@@ -118,7 +153,7 @@ public class MetadataQuerierByFileImpl implements IMetadataQuerier {
}
List<ITimeSeriesMetadata> timeseriesMetaDataList =
- tsFileReader.readITimeseriesMetadata(selectedDevice, selectedMeasurements);
+ tsFileReader.readITimeseriesMetadata(selectedDevice, selectedMeasurements, null, false);
for (ITimeSeriesMetadata timeseriesMetadata : timeseriesMetaDataList) {
List<IChunkMetadata> chunkMetadataList =
tsFileReader.readIChunkMetaDataList(timeseriesMetadata);
@@ -132,8 +167,8 @@ public class MetadataQuerierByFileImpl implements IMetadataQuerier {
} else {
measurementId = ((TimeseriesMetadata) timeseriesMetadata).getMeasurementId();
}
- this.chunkMetaDataCache.put(
- new Path(selectedDevice, measurementId, true), chunkMetadataList);
+ this.deviceIdChunkMetadataCache.put(
+ new Pair<>(selectedDevice, measurementId), chunkMetadataList);
count += chunkMetadataList.size();
if (count == CACHED_ENTRY_NUMBER) {
enough = true;
@@ -156,6 +191,10 @@ public class MetadataQuerierByFileImpl implements IMetadataQuerier {
return tsFileReader.getIChunkMetadataList(path);
}
+ private List<IChunkMetadata> loadChunkMetadata(Pair<IDeviceID, String> key) throws IOException {
+ return tsFileReader.getIChunkMetadataList(key.getLeft(), key.right);
+ }
+
@Override
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning
public List<TimeRange> convertSpace2TimePartition(
@@ -253,6 +292,12 @@ public class MetadataQuerierByFileImpl implements IMetadataQuerier {
@Override
public void clear() {
- chunkMetaDataCache.clear();
+ deviceIdChunkMetadataCache.clear();
+ }
+
+ @Override
+ public Iterator<Pair<IDeviceID, MetadataIndexNode>> deviceIterator(
+ MetadataIndexNode root, ExpressionTree idFilter) {
+ return new DeviceMetaIterator(tsFileReader, root, idFilter);
}
}
diff --git a/cpp/examples/c_examples/c_examples.h b/java/tsfile/src/main/java/org/apache/tsfile/read/expression/ExpressionTree.java
similarity index 81%
copy from cpp/examples/c_examples/c_examples.h
copy to java/tsfile/src/main/java/org/apache/tsfile/read/expression/ExpressionTree.java
index d29090cb..fa0b1138 100644
--- a/cpp/examples/c_examples/c_examples.h
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/expression/ExpressionTree.java
@@ -17,16 +17,12 @@
* under the License.
*/
+package org.apache.tsfile.read.expression;
-#include "cwrapper/TsFile-cwrapper.h"
+import org.apache.tsfile.read.filter.basic.Filter;
-#ifdef __cplusplus
-extern "C" {
-#endif
+public interface ExpressionTree {
+ boolean satisfy(Object value);
-ErrorCode write_tsfile();
-ErrorCode read_tsfile();
-
-#ifdef __cplusplus
+ Filter toFilter();
}
-#endif
\ No newline at end of file
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/factory/FilterFactory.java b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/factory/FilterFactory.java
index 1fa9b4e0..f29fc7ed 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/factory/FilterFactory.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/factory/FilterFactory.java
@@ -24,6 +24,8 @@ import org.apache.tsfile.read.filter.operator.And;
import org.apache.tsfile.read.filter.operator.Not;
import org.apache.tsfile.read.filter.operator.Or;
+import java.util.List;
+
import static org.apache.tsfile.utils.Preconditions.checkArgument;
public class FilterFactory {
@@ -43,6 +45,14 @@ public class FilterFactory {
return new And(left, right);
}
+ public static Filter and(List<Filter> filterList) {
+ And result = new And(filterList.get(0), filterList.get(1));
+ for (int i = 2, size = filterList.size(); i < size; i++) {
+ result = new And(result, filterList.get(i));
+ }
+ return result;
+ }
+
public static Filter or(Filter left, Filter right) {
if (left == null && right == null) {
return null;
@@ -54,6 +64,14 @@ public class FilterFactory {
return new Or(left, right);
}
+ public static Filter or(List<Filter> filterList) {
+ Or result = new Or(filterList.get(0), filterList.get(1));
+ for (int i = 2, size = filterList.size(); i < size; i++) {
+ result = new Or(result, filterList.get(i));
+ }
+ return result;
+ }
+
public static Not not(Filter filter) {
checkArgument(filter != null, "filter cannot be null");
return new Not(filter);
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/query/executor/TableQueryExecutor.java b/java/tsfile/src/main/java/org/apache/tsfile/read/query/executor/TableQueryExecutor.java
new file mode 100644
index 00000000..9cce7c78
--- /dev/null
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/query/executor/TableQueryExecutor.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tsfile.read.query.executor;
+
+import org.apache.tsfile.exception.read.NoColumnException;
+import org.apache.tsfile.exception.read.ReadProcessException;
+import org.apache.tsfile.exception.read.UnsupportedOrderingException;
+import org.apache.tsfile.file.metadata.MetadataIndexNode;
+import org.apache.tsfile.file.metadata.TableSchema;
+import org.apache.tsfile.file.metadata.TsFileMetadata;
+import org.apache.tsfile.read.controller.IChunkLoader;
+import org.apache.tsfile.read.controller.IMetadataQuerier;
+import org.apache.tsfile.read.expression.ExpressionTree;
+import org.apache.tsfile.read.query.executor.task.DeviceTaskIterator;
+import org.apache.tsfile.read.reader.block.DeviceOrderedTsBlockReader;
+import org.apache.tsfile.read.reader.block.TsBlockReader;
+import org.apache.tsfile.read.reader.block.TsBlockReader.EmptyTsBlockReader;
+import org.apache.tsfile.write.record.Tablet.ColumnType;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class TableQueryExecutor {
+
+ private IMetadataQuerier metadataQuerier;
+ private IChunkLoader chunkLoader;
+ private TableQueryOrdering tableQueryOrdering;
+ private int blockSize = 1024;
+
+ public TableQueryExecutor(
+ IMetadataQuerier metadataQuerier,
+ IChunkLoader chunkLoader,
+ TableQueryOrdering tableQueryOrdering) {
+ this.metadataQuerier = metadataQuerier;
+ this.chunkLoader = chunkLoader;
+ this.tableQueryOrdering = tableQueryOrdering;
+ }
+
+ /**
+ * @param tableName table to query
+ * @param columns columns to query (ID or MEASUREMENT)
+ * @param timeFilter time predicate
+ * @param idFilter id predicate
+ * @param measurementFilter measurement predicate
+ * @return an iterator of TsBlocks
+ * @throws ReadProcessException if the read process fails
+ */
+ public TsBlockReader query(
+ String tableName,
+ List<String> columns,
+ ExpressionTree timeFilter,
+ ExpressionTree idFilter,
+ ExpressionTree measurementFilter)
+ throws ReadProcessException {
+ TsFileMetadata fileMetadata = metadataQuerier.getWholeFileMetadata();
+ MetadataIndexNode tableRoot = fileMetadata.getTableMetadataIndexNode(tableName);
+ TableSchema tableSchema = fileMetadata.getTableSchemaMap().get(tableName);
+ if (tableRoot == null || tableSchema == null) {
+ return new EmptyTsBlockReader();
+ }
+
+ ColumnMapping columnMapping = new ColumnMapping();
+ for (int i = 0; i < columns.size(); i++) {
+ String column = columns.get(i);
+ columnMapping.add(column, i, tableSchema);
+ }
+ columnMapping.add(measurementFilter);
+
+ DeviceTaskIterator deviceTaskIterator =
+ new DeviceTaskIterator(
+ columns, tableRoot, columnMapping, metadataQuerier, idFilter, tableSchema);
+ switch (tableQueryOrdering) {
+ case DEVICE:
+ return new DeviceOrderedTsBlockReader(
+ deviceTaskIterator,
+ metadataQuerier,
+ chunkLoader,
+ timeFilter,
+ measurementFilter,
+ blockSize);
+ case TIME:
+ default:
+ throw new UnsupportedOrderingException(tableQueryOrdering.toString());
+ }
+ }
+
+ public class ColumnMapping {
+ /**
+ * The same column may occur multiple times in a query, but we surely do not want to read it
+ * redundantly. This mapping is used to put data of the same series into multiple columns.
+ */
+ private Map<String, List<Integer>> columnPosMap = new HashMap<>();
+
+ private Set<String> idColumns = new HashSet<>();
+ private Set<String> measurementColumns = new HashSet<>();
+
+ public void add(String columnName, int i, TableSchema schema) throws NoColumnException {
+ final int columnIndex = schema.findColumnIndex(columnName);
+ if (columnIndex < 0) {
+ throw new NoColumnException(columnName);
+ }
+
+ final ColumnType columnType = schema.getColumnTypes().get(columnIndex);
+ columnPosMap.computeIfAbsent(columnName, k -> new ArrayList<>()).add(i);
+ if (columnType.equals(ColumnType.ID)) {
+ idColumns.add(columnName);
+ } else {
+ measurementColumns.add(columnName);
+ }
+ }
+
+ public void add(ExpressionTree measurementFilter) {
+ // TODO: get measurements in the filter and add them to measurementColumns
+ }
+
+ public List<Integer> getColumnPos(String columnName) {
+ return columnPosMap.getOrDefault(columnName, Collections.emptyList());
+ }
+
+ public boolean isId(String columnName) {
+ return idColumns.contains(columnName);
+ }
+
+ public boolean isMeasurement(String columnName) {
+ return measurementColumns.contains(columnName);
+ }
+
+ public Set<String> getIdColumns() {
+ return idColumns;
+ }
+
+ public Set<String> getMeasurementColumns() {
+ return measurementColumns;
+ }
+ }
+
+ public enum TableQueryOrdering {
+ TIME,
+ DEVICE
+ }
+}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/query/executor/task/DeviceQueryTask.java b/java/tsfile/src/main/java/org/apache/tsfile/read/query/executor/task/DeviceQueryTask.java
new file mode 100644
index 00000000..e88218e6
--- /dev/null
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/query/executor/task/DeviceQueryTask.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tsfile.read.query.executor.task;
+
+import org.apache.tsfile.file.metadata.IDeviceID;
+import org.apache.tsfile.file.metadata.MetadataIndexNode;
+import org.apache.tsfile.file.metadata.TableSchema;
+import org.apache.tsfile.read.query.executor.TableQueryExecutor.ColumnMapping;
+
+import java.util.List;
+
+public class DeviceQueryTask {
+ private final IDeviceID deviceID;
+ private final List<String> columnNames;
+ private final ColumnMapping columnMapping;
+ private final MetadataIndexNode indexRoot;
+ private final TableSchema tableSchema;
+
+ public DeviceQueryTask(
+ IDeviceID deviceID,
+ List<String> columnNames,
+ ColumnMapping columnMapping,
+ MetadataIndexNode indexRoot,
+ TableSchema tableSchema) {
+ this.deviceID = deviceID;
+ this.columnNames = columnNames;
+ this.columnMapping = columnMapping;
+ this.indexRoot = indexRoot;
+ this.tableSchema = tableSchema;
+ }
+
+ public IDeviceID getDeviceID() {
+ return deviceID;
+ }
+
+ public List<String> getColumnNames() {
+ return columnNames;
+ }
+
+ public ColumnMapping getColumnMapping() {
+ return columnMapping;
+ }
+
+ public MetadataIndexNode getIndexRoot() {
+ return indexRoot;
+ }
+
+ public TableSchema getTableSchema() {
+ return tableSchema;
+ }
+
+ @Override
+ public String toString() {
+ return "DeviceQueryTask{" + "deviceID=" + deviceID + ", columnNames=" + columnNames + '}';
+ }
+}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/query/executor/task/DeviceTaskIterator.java b/java/tsfile/src/main/java/org/apache/tsfile/read/query/executor/task/DeviceTaskIterator.java
new file mode 100644
index 00000000..8ee44650
--- /dev/null
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/query/executor/task/DeviceTaskIterator.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tsfile.read.query.executor.task;
+
+import org.apache.tsfile.file.metadata.IDeviceID;
+import org.apache.tsfile.file.metadata.MetadataIndexNode;
+import org.apache.tsfile.file.metadata.TableSchema;
+import org.apache.tsfile.read.controller.IMetadataQuerier;
+import org.apache.tsfile.read.expression.ExpressionTree;
+import org.apache.tsfile.read.query.executor.TableQueryExecutor.ColumnMapping;
+import org.apache.tsfile.utils.Pair;
+
+import java.util.Iterator;
+import java.util.List;
+
+public class DeviceTaskIterator implements Iterator<DeviceQueryTask> {
+ private List<String> columnNames;
+ private ColumnMapping columnMapping;
+ private TableSchema tableSchema;
+ private Iterator<Pair<IDeviceID, MetadataIndexNode>> deviceMetaIterator;
+
+ public DeviceTaskIterator(
+ List<String> columnNames,
+ MetadataIndexNode indexRoot,
+ ColumnMapping columnMapping,
+ IMetadataQuerier metadataQuerier,
+ ExpressionTree idFilter,
+ TableSchema tableSchema) {
+ this.columnNames = columnNames;
+ this.columnMapping = columnMapping;
+ this.deviceMetaIterator = metadataQuerier.deviceIterator(indexRoot, idFilter);
+ this.tableSchema = tableSchema;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return deviceMetaIterator.hasNext();
+ }
+
+ @Override
+ public DeviceQueryTask next() {
+ final Pair<IDeviceID, MetadataIndexNode> next = deviceMetaIterator.next();
+ return new DeviceQueryTask(next.left, columnNames, columnMapping, next.right, tableSchema);
+ }
+}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/block/DeviceOrderedTsBlockReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/block/DeviceOrderedTsBlockReader.java
new file mode 100644
index 00000000..30ddca8f
--- /dev/null
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/block/DeviceOrderedTsBlockReader.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tsfile.read.reader.block;
+
+import org.apache.tsfile.read.common.block.TsBlock;
+import org.apache.tsfile.read.controller.IChunkLoader;
+import org.apache.tsfile.read.controller.IMetadataQuerier;
+import org.apache.tsfile.read.expression.ExpressionTree;
+import org.apache.tsfile.read.query.executor.task.DeviceQueryTask;
+import org.apache.tsfile.read.query.executor.task.DeviceTaskIterator;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.NoSuchElementException;
+
+public class DeviceOrderedTsBlockReader implements TsBlockReader {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(DeviceOrderedTsBlockReader.class);
+ private final DeviceTaskIterator taskIterator;
+ private final IMetadataQuerier metadataQuerier;
+ private final IChunkLoader chunkLoader;
+ private final int blockSize;
+ private SingleDeviceTsBlockReader currentReader;
+ private ExpressionTree timeFilter;
+ private ExpressionTree measurementFilter;
+
+ public DeviceOrderedTsBlockReader(
+ DeviceTaskIterator taskIterator,
+ IMetadataQuerier metadataQuerier,
+ IChunkLoader chunkLoader,
+ ExpressionTree timeFilter,
+ ExpressionTree measurementFilter,
+ int blockSize) {
+ this.taskIterator = taskIterator;
+ this.metadataQuerier = metadataQuerier;
+ this.chunkLoader = chunkLoader;
+ this.blockSize = blockSize;
+ this.timeFilter = timeFilter;
+ this.measurementFilter = measurementFilter;
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (currentReader != null && currentReader.hasNext()) {
+ return true;
+ }
+ while (taskIterator.hasNext()) {
+ final DeviceQueryTask nextTask = taskIterator.next();
+ try {
+ currentReader =
+ new SingleDeviceTsBlockReader(
+ nextTask, metadataQuerier, chunkLoader, blockSize, timeFilter, measurementFilter);
+ } catch (IOException e) {
+ LOGGER.error("Failed to construct reader for {}", nextTask, e);
+ }
+ if (currentReader.hasNext()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public TsBlock next() throws IOException {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ return currentReader.next();
+ }
+
+ @Override
+ public void close() throws Exception {
+ if (currentReader != null) {
+ currentReader.close();
+ }
+ }
+}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/block/SingleDeviceTsBlockReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/block/SingleDeviceTsBlockReader.java
new file mode 100644
index 00000000..473c3127
--- /dev/null
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/block/SingleDeviceTsBlockReader.java
@@ -0,0 +1,408 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tsfile.read.reader.block;
+
+import org.apache.tsfile.block.column.Column;
+import org.apache.tsfile.file.metadata.AlignedChunkMetadata;
+import org.apache.tsfile.file.metadata.IChunkMetadata;
+import org.apache.tsfile.read.common.BatchData;
+import org.apache.tsfile.read.common.block.TsBlock;
+import org.apache.tsfile.read.controller.IChunkLoader;
+import org.apache.tsfile.read.controller.IMetadataQuerier;
+import org.apache.tsfile.read.expression.ExpressionTree;
+import org.apache.tsfile.read.filter.basic.Filter;
+import org.apache.tsfile.read.query.executor.task.DeviceQueryTask;
+import org.apache.tsfile.read.reader.series.AbstractFileSeriesReader;
+import org.apache.tsfile.read.reader.series.FileSeriesReader;
+import org.apache.tsfile.utils.Binary;
+import org.apache.tsfile.utils.TsPrimitiveType;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.NoSuchElementException;
+
+public class SingleDeviceTsBlockReader implements TsBlockReader {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(SingleDeviceTsBlockReader.class);
+ private final DeviceQueryTask task;
+ private final ExpressionTree measurementExpression;
+ private final int blockSize;
+
+ private final TsBlock currentBlock;
+ private boolean lastBlockReturned = true;
+ private final Map<String, MeasurementColumnContext> measureColumnContextMap;
+ private final Map<String, IdColumnContext> idColumnContextMap;
+
+ private long nextTime;
+
+ public SingleDeviceTsBlockReader(
+ DeviceQueryTask task,
+ IMetadataQuerier metadataQuerier,
+ IChunkLoader chunkLoader,
+ int blockSize,
+ ExpressionTree timeExpression,
+ ExpressionTree measurementFilter)
+ throws IOException {
+ this.task = task;
+ this.blockSize = blockSize;
+ this.measurementExpression = measurementFilter;
+
+ this.currentBlock =
+ TsBlock.buildTsBlock(task.getColumnNames(), task.getTableSchema(), blockSize);
+ this.measureColumnContextMap = new HashMap<>();
+ this.idColumnContextMap = new HashMap<>();
+
+ final List<List<IChunkMetadata>> chunkMetadataLists =
+ metadataQuerier.getChunkMetadataLists(
+ task.getDeviceID(),
+ task.getColumnMapping().getMeasurementColumns(),
+ task.getIndexRoot());
+
+ Filter timeFilter = timeExpression == null ? null : timeExpression.toFilter();
+ for (List<IChunkMetadata> chunkMetadataList : chunkMetadataLists) {
+ constructColumnContext(chunkMetadataList, chunkLoader, timeFilter);
+ }
+
+ for (String idColumn : task.getColumnMapping().getIdColumns()) {
+ final List<Integer> columnPosInResult = task.getColumnMapping().getColumnPos(idColumn);
+ // the first segment in DeviceId is the table name
+ final int columnPosInId = task.getTableSchema().findColumnIndex(idColumn) + 1;
+ idColumnContextMap.put(idColumn, new IdColumnContext(columnPosInResult, columnPosInId));
+ }
+ }
+
+ private void constructColumnContext(
+ List<IChunkMetadata> chunkMetadataList, IChunkLoader chunkLoader, Filter timeFilter)
+ throws IOException {
+ if (chunkMetadataList.isEmpty()) {
+ return;
+ }
+ final IChunkMetadata chunkMetadata = chunkMetadataList.get(0);
+ AbstractFileSeriesReader seriesReader =
+ new FileSeriesReader(chunkLoader, chunkMetadataList, timeFilter);
+ if (seriesReader.hasNextBatch()) {
+ if (chunkMetadata instanceof AlignedChunkMetadata) {
+ final List<String> currentChunkMeasurementNames =
+ seriesReader.getCurrentChunkMeasurementNames();
+ List<List<Integer>> posInResult = new ArrayList<>();
+ for (String currentChunkMeasurementName : currentChunkMeasurementNames) {
+ posInResult.add(task.getColumnMapping().getColumnPos(currentChunkMeasurementName));
+ }
+ measureColumnContextMap.put(
+ "",
+ new VectorMeasurementColumnContext(
+ posInResult, seriesReader.nextBatch(), seriesReader));
+ } else {
+ final String measurementUid = chunkMetadata.getMeasurementUid();
+ measureColumnContextMap.put(
+ measurementUid,
+ new SingleMeasurementColumnContext(
+ measurementUid,
+ task.getColumnMapping().getColumnPos(measurementUid),
+ seriesReader.nextBatch(),
+ seriesReader));
+ }
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (!lastBlockReturned) {
+ return true;
+ }
+
+ if (measureColumnContextMap.isEmpty()) {
+ return false;
+ }
+
+ currentBlock.reset();
+ nextTime = Long.MAX_VALUE;
+ List<MeasurementColumnContext> minTimeColumns = new ArrayList<>();
+
+ while (currentBlock.getPositionCount() < blockSize) {
+ // find the minimum time among the batches and the associated columns
+ for (Entry<String, MeasurementColumnContext> entry : measureColumnContextMap.entrySet()) {
+ final BatchData batchData = entry.getValue().currentBatch;
+ final long currentTime = batchData.currentTime();
+ if (nextTime > currentTime) {
+ nextTime = currentTime;
+ minTimeColumns.clear();
+ minTimeColumns.add(entry.getValue());
+ } else if (nextTime == currentTime) {
+ minTimeColumns.add(entry.getValue());
+ }
+ }
+
+ try {
+ fillMeasurements(minTimeColumns);
+ } catch (IOException e) {
+ LOGGER.error("Cannot fill measurements", e);
+ return false;
+ }
+
+ // all columns have exhausted
+ if (measureColumnContextMap.isEmpty()) {
+ break;
+ }
+ }
+
+ if (currentBlock.getPositionCount() > 0) {
+ fillIds();
+ currentBlock.fillTrailingNulls();
+ lastBlockReturned = false;
+ return true;
+ }
+
+ return false;
+ }
+
+ private void fillIds() {
+ for (Entry<String, IdColumnContext> entry : idColumnContextMap.entrySet()) {
+ final IdColumnContext idColumnContext = entry.getValue();
+ for (Integer pos : idColumnContext.posInResult) {
+ final Column column = currentBlock.getColumn(pos);
+ fillIdColumn(
+ column,
+ task.getDeviceID().segment(idColumnContext.posInDeviceId),
+ 0,
+ currentBlock.getPositionCount());
+ }
+ }
+ }
+
+ private void fillMeasurements(List<MeasurementColumnContext> minTimeColumns) throws IOException {
+ if (measurementExpression == null || measurementExpression.satisfy(this)) {
+ // use the time to fill the block
+ final int positionCount = currentBlock.getPositionCount();
+ currentBlock.getTimeColumn().getLongs()[positionCount] = nextTime;
+ // project the value columns to the result
+ for (final MeasurementColumnContext columnContext : minTimeColumns) {
+ columnContext.fillInto(currentBlock, positionCount);
+ advanceColumn(columnContext.currentBatch, columnContext);
+ }
+ currentBlock.setPositionCount(positionCount + 1);
+ } else {
+ for (final MeasurementColumnContext columnContext : minTimeColumns) {
+ final BatchData batchData = columnContext.currentBatch;
+ advanceColumn(batchData, columnContext);
+ }
+ }
+ }
+
+ private void advanceColumn(BatchData batchData, MeasurementColumnContext columnContext)
+ throws IOException {
+ batchData.next();
+ if (!batchData.hasCurrent()) {
+ // get next batch of the column
+ if (columnContext.seriesReader.hasNextBatch()) {
+ columnContext.currentBatch = columnContext.seriesReader.nextBatch();
+ } else {
+ // no more data in this column
+ columnContext.removeFrom(measureColumnContextMap);
+ }
+ }
+ }
+
+ private void fillIdColumn(Column column, Object val, int startPos, int endPos) {
+ switch (column.getDataType()) {
+ case TEXT:
+ if (val instanceof String) {
+ val = new Binary(((String) val), StandardCharsets.UTF_8);
+ }
+ Arrays.fill(column.getBinaries(), startPos, endPos, val);
+ break;
+ case BOOLEAN:
+ Arrays.fill(column.getBooleans(), startPos, endPos, ((boolean) val));
+ break;
+ case INT32:
+ Arrays.fill(column.getInts(), startPos, endPos, ((int) val));
+ break;
+ case INT64:
+ Arrays.fill(column.getLongs(), startPos, endPos, ((long) val));
+ break;
+ case FLOAT:
+ Arrays.fill(column.getFloats(), startPos, endPos, ((float) val));
+ break;
+ case DOUBLE:
+ Arrays.fill(column.getDoubles(), startPos, endPos, ((double) val));
+ break;
+ default:
+ throw new IllegalArgumentException("Unsupported data type: " + column.getDataType());
+ }
+ column.setPositionCount(endPos);
+ }
+
+ private static void fillSingleMeasurementColumn(Column column, BatchData batchData, int pos) {
+ switch (batchData.getDataType()) {
+ case BOOLEAN:
+ column.getBooleans()[pos] = batchData.getBoolean();
+ break;
+ case DOUBLE:
+ column.getDoubles()[pos] = batchData.getDouble();
+ break;
+ case FLOAT:
+ column.getFloats()[pos] = batchData.getFloat();
+ break;
+ case INT32:
+ column.getInts()[pos] = batchData.getInt();
+ break;
+ case TEXT:
+ column.getBinaries()[pos] = batchData.getBinary();
+ break;
+ case INT64:
+ column.getLongs()[pos] = batchData.getLong();
+ break;
+ default:
+ throw new IllegalArgumentException("Unsupported data type: " + batchData.getDataType());
+ }
+ column.setPositionCount(pos + 1);
+ }
+
+ @Override
+ public TsBlock next() throws IOException {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ lastBlockReturned = true;
+ return currentBlock;
+ }
+
+ @Override
+ public void close() throws Exception {
+ // nothing to be done
+ }
+
+ public abstract static class MeasurementColumnContext {
+
+ protected BatchData currentBatch;
+ protected final AbstractFileSeriesReader seriesReader;
+
+ protected MeasurementColumnContext(
+ AbstractFileSeriesReader seriesReader, BatchData currentBatch) {
+ this.seriesReader = seriesReader;
+ this.currentBatch = currentBatch;
+ }
+
+ abstract void removeFrom(Map<String, MeasurementColumnContext> columnContextMap);
+
+ abstract void fillInto(TsBlock block, int position);
+ }
+
+ // gather necessary fields in this class to avoid redundant map access
+ public static class SingleMeasurementColumnContext extends MeasurementColumnContext {
+
+ private final String columnName;
+ private final List<Integer> posInResult;
+
+ public SingleMeasurementColumnContext(
+ String columnName,
+ List<Integer> posInResult,
+ BatchData currentBatch,
+ AbstractFileSeriesReader seriesReader) {
+ super(seriesReader, currentBatch);
+ this.columnName = columnName;
+ this.posInResult = posInResult;
+ }
+
+ @Override
+ void removeFrom(Map<String, MeasurementColumnContext> columnContextMap) {
+ columnContextMap.remove(columnName);
+ }
+
+ @Override
+ void fillInto(TsBlock block, int position) {
+ for (Integer pos : posInResult) {
+ final Column column = block.getColumn(pos);
+ fillSingleMeasurementColumn(column, currentBatch, position);
+ }
+ }
+ }
+
+ public static class VectorMeasurementColumnContext extends MeasurementColumnContext {
+
+ private final List<List<Integer>> posInResult;
+
+ public VectorMeasurementColumnContext(
+ List<List<Integer>> posInResult,
+ BatchData currentBatch,
+ AbstractFileSeriesReader seriesReader) {
+ super(seriesReader, currentBatch);
+ this.posInResult = posInResult;
+ }
+
+ @Override
+ void removeFrom(Map<String, MeasurementColumnContext> columnContextMap) {
+ columnContextMap.remove("");
+ }
+
+ @Override
+ void fillInto(TsBlock block, int blockRowNum) {
+ final TsPrimitiveType[] vector = currentBatch.getVector();
+ for (int i = 0; i < vector.length; i++) {
+ final TsPrimitiveType value = vector[i];
+ final List<Integer> columnPositions = posInResult.get(i);
+ for (Integer pos : columnPositions) {
+ switch (value.getDataType()) {
+ case TEXT:
+ block.getColumn(pos).getBinaries()[blockRowNum] = value.getBinary();
+ break;
+ case INT32:
+ block.getColumn(pos).getInts()[blockRowNum] = value.getInt();
+ break;
+ case INT64:
+ block.getColumn(pos).getLongs()[blockRowNum] = value.getLong();
+ break;
+ case BOOLEAN:
+ block.getColumn(pos).getBooleans()[blockRowNum] = value.getBoolean();
+ break;
+ case FLOAT:
+ block.getColumn(pos).getFloats()[blockRowNum] = value.getFloat();
+ break;
+ case DOUBLE:
+ block.getColumn(pos).getDoubles()[blockRowNum] = value.getDouble();
+ break;
+ default:
+ throw new IllegalArgumentException("Unsupported data type: " + value.getDataType());
+ }
+ }
+ }
+ }
+ }
+
+ public static class IdColumnContext {
+
+ private final List<Integer> posInResult;
+ private final int posInDeviceId;
+
+ public IdColumnContext(List<Integer> posInResult, int posInDeviceId) {
+ this.posInResult = posInResult;
+ this.posInDeviceId = posInDeviceId;
+ }
+ }
+}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/IMetadataIndexEntry.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/block/TsBlockReader.java
similarity index 62%
copy from java/tsfile/src/main/java/org/apache/tsfile/file/IMetadataIndexEntry.java
copy to java/tsfile/src/main/java/org/apache/tsfile/read/reader/block/TsBlockReader.java
index cc1cea3b..a63c0717 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/file/IMetadataIndexEntry.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/block/TsBlockReader.java
@@ -17,20 +17,32 @@
* under the License.
*/
-package org.apache.tsfile.file;
+package org.apache.tsfile.read.reader.block;
+
+import org.apache.tsfile.read.common.block.TsBlock;
import java.io.IOException;
-import java.io.OutputStream;
-public interface IMetadataIndexEntry {
+public interface TsBlockReader extends AutoCloseable {
+ boolean hasNext();
- long getOffset();
+ TsBlock next() throws IOException;
- void setOffset(long offset);
+ class EmptyTsBlockReader implements TsBlockReader {
- int serializeTo(OutputStream outputStream) throws IOException;
+ @Override
+ public boolean hasNext() {
+ return false;
+ }
- Comparable getCompareKey();
+ @Override
+ public TsBlock next() throws IOException {
+ return null;
+ }
- boolean isDeviceLevel();
+ @Override
+ public void close() throws Exception {
+ // nothing to be done
+ }
+ }
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/series/AbstractFileSeriesReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/series/AbstractFileSeriesReader.java
index edb8c05e..735b44c9 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/series/AbstractFileSeriesReader.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/series/AbstractFileSeriesReader.java
@@ -27,6 +27,7 @@ import org.apache.tsfile.read.reader.IBatchReader;
import org.apache.tsfile.read.reader.IChunkReader;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.List;
/** Series reader is used to query one series of one tsfile. */
@@ -35,6 +36,7 @@ public abstract class AbstractFileSeriesReader implements IBatchReader {
protected IChunkLoader chunkLoader;
protected List<IChunkMetadata> chunkMetadataList;
protected IChunkReader chunkReader;
+ protected List<String> currentChunkMeasurementNames = new ArrayList<>();
private int chunkToRead;
protected Filter filter;
@@ -91,4 +93,8 @@ public abstract class AbstractFileSeriesReader implements IBatchReader {
private IChunkMetadata nextChunkMeta() {
return chunkMetadataList.get(chunkToRead++);
}
+
+ public List<String> getCurrentChunkMeasurementNames() {
+ return currentChunkMeasurementNames;
+ }
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/series/FileSeriesReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/series/FileSeriesReader.java
index 1704d553..c6cb00d9 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/series/FileSeriesReader.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/series/FileSeriesReader.java
@@ -45,9 +45,11 @@ public class FileSeriesReader extends AbstractFileSeriesReader {
@Override
protected void initChunkReader(IChunkMetadata chunkMetaData) throws IOException {
+ currentChunkMeasurementNames.clear();
if (chunkMetaData instanceof ChunkMetadata) {
Chunk chunk = chunkLoader.loadChunk((ChunkMetadata) chunkMetaData);
this.chunkReader = new ChunkReader(chunk, filter);
+ currentChunkMeasurementNames.add(chunkMetaData.getMeasurementUid());
} else {
AlignedChunkMetadata alignedChunkMetadata = (AlignedChunkMetadata) chunkMetaData;
Chunk timeChunk =
@@ -55,6 +57,7 @@ public class FileSeriesReader extends AbstractFileSeriesReader {
List<Chunk> valueChunkList = new ArrayList<>();
for (IChunkMetadata metadata : alignedChunkMetadata.getValueChunkMetadataList()) {
valueChunkList.add(chunkLoader.loadChunk((ChunkMetadata) metadata));
+ currentChunkMeasurementNames.add(metadata.getMeasurementUid());
}
this.chunkReader = new AlignedChunkReader(timeChunk, valueChunkList, filter);
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/MeasurementGroup.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/MeasurementGroup.java
index 996e3476..84f95f24 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/utils/MeasurementGroup.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/MeasurementGroup.java
@@ -18,7 +18,7 @@
*/
package org.apache.tsfile.utils;
-import org.apache.tsfile.write.schema.MeasurementSchema;
+import org.apache.tsfile.write.schema.IMeasurementSchema;
import java.io.Serializable;
import java.util.HashMap;
@@ -27,22 +27,22 @@ import java.util.Map;
public class MeasurementGroup implements Serializable {
private boolean isAligned;
- private Map<String, MeasurementSchema> measurementSchemaMap;
+ private Map<String, IMeasurementSchema> measurementSchemaMap;
public MeasurementGroup(boolean isAligned) {
this.isAligned = isAligned;
measurementSchemaMap = new HashMap<>();
}
- public MeasurementGroup(boolean isAligned, List<MeasurementSchema> measurementSchemas) {
+ public MeasurementGroup(boolean isAligned, List<IMeasurementSchema> measurementSchemas) {
this.isAligned = isAligned;
measurementSchemaMap = new HashMap<>();
- for (MeasurementSchema schema : measurementSchemas) {
+ for (IMeasurementSchema schema : measurementSchemas) {
measurementSchemaMap.put(schema.getMeasurementId(), schema);
}
}
- public MeasurementGroup(boolean isAligned, Map<String, MeasurementSchema> measurementSchemaMap) {
+ public MeasurementGroup(boolean isAligned, Map<String, IMeasurementSchema> measurementSchemaMap) {
this.isAligned = isAligned;
this.measurementSchemaMap = measurementSchemaMap;
}
@@ -55,11 +55,11 @@ public class MeasurementGroup implements Serializable {
isAligned = aligned;
}
- public Map<String, MeasurementSchema> getMeasurementSchemaMap() {
+ public Map<String, IMeasurementSchema> getMeasurementSchemaMap() {
return measurementSchemaMap;
}
- public void setMeasurementSchemaMap(Map<String, MeasurementSchema> measurementSchemaMap) {
+ public void setMeasurementSchemaMap(Map<String, IMeasurementSchema> measurementSchemaMap) {
this.measurementSchemaMap = measurementSchemaMap;
}
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteForEncodingUtils.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteForEncodingUtils.java
index 7d741223..15940db5 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteForEncodingUtils.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteForEncodingUtils.java
@@ -18,6 +18,8 @@
*/
package org.apache.tsfile.utils;
+import org.apache.tsfile.common.conf.TSFileConfig;
+
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
@@ -237,6 +239,16 @@ public class ReadWriteForEncodingUtils {
return position;
}
+ public static int varIntStringSize(String str) {
+ final byte[] bytes = str.getBytes(TSFileConfig.STRING_CHARSET);
+ return varIntSize(bytes.length) + bytes.length;
+ }
+
+ public static int intStringSize(String str) {
+ final byte[] bytes = str.getBytes(TSFileConfig.STRING_CHARSET);
+ return Integer.BYTES + bytes.length;
+ }
+
/**
* Returns the encoding size in bytes of its input value.
*
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteIOUtils.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteIOUtils.java
index 6e55e760..e4133cc7 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteIOUtils.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteIOUtils.java
@@ -64,7 +64,7 @@ public class ReadWriteIOUtils {
public static final int FLOAT_LEN = 4;
public static final float BIT_LEN = 0.125F;
- private static final int NO_BYTE_TO_READ = -1;
+ public static final int NO_BYTE_TO_READ = -1;
private static final byte[] magicStringBytes;
@@ -378,7 +378,7 @@ public class ReadWriteIOUtils {
return len;
}
- byte[] bytes = s.getBytes();
+ byte[] bytes = s.getBytes(TSFileConfig.STRING_CHARSET);
len += write(bytes.length, outputStream);
outputStream.write(bytes);
len += bytes.length;
@@ -414,7 +414,7 @@ public class ReadWriteIOUtils {
return write(NO_BYTE_TO_READ, buffer);
}
int len = 0;
- byte[] bytes = s.getBytes();
+ byte[] bytes = s.getBytes(TSFileConfig.STRING_CHARSET);
len += write(bytes.length, buffer);
buffer.put(bytes);
len += bytes.length;
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileGeneratorUtils.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileGeneratorUtils.java
index ba1b5e31..ff7cb33c 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileGeneratorUtils.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileGeneratorUtils.java
@@ -57,7 +57,7 @@ public class TsFileGeneratorUtils {
public static void writeWithTsRecord(
TsFileWriter tsFileWriter,
String deviceId,
- List<MeasurementSchema> schemas,
+ List<IMeasurementSchema> schemas,
long rowSize,
long startTime,
long startValue,
@@ -110,7 +110,7 @@ public class TsFileGeneratorUtils {
public static void writeWithTablet(
TsFileWriter tsFileWriter,
String deviceId,
- List<MeasurementSchema> schemas,
+ List<IMeasurementSchema> schemas,
long rowNum,
long startTime,
long startValue,
@@ -173,7 +173,7 @@ public class TsFileGeneratorUtils {
TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(pageSize);
try (TsFileWriter tsFileWriter = new TsFileWriter(file)) {
// register align timeseries
- List<MeasurementSchema> alignedMeasurementSchemas = new ArrayList<>();
+ List<IMeasurementSchema> alignedMeasurementSchemas = new ArrayList<>();
for (int i = 0; i < measurementNum; i++) {
alignedMeasurementSchemas.add(
new MeasurementSchema("s" + i, TSDataType.INT64, TSEncoding.PLAIN));
@@ -196,7 +196,7 @@ public class TsFileGeneratorUtils {
}
// register nonAlign timeseries
- List<MeasurementSchema> measurementSchemas = new ArrayList<>();
+ List<IMeasurementSchema> measurementSchemas = new ArrayList<>();
for (int i = 0; i < measurementNum; i++) {
measurementSchemas.add(
new MeasurementSchema("s" + i, TSDataType.INT64, TSEncoding.PLAIN));
@@ -245,7 +245,7 @@ public class TsFileGeneratorUtils {
TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(pageSize);
try (TsFileWriter tsFileWriter = new TsFileWriter(file)) {
// register align timeseries
- List<MeasurementSchema> alignedMeasurementSchemas = new ArrayList<>();
+ List<IMeasurementSchema> alignedMeasurementSchemas = new ArrayList<>();
for (int i = 0; i < measurementNum; i++) {
alignedMeasurementSchemas.add(
new MeasurementSchema("s" + i, getDataType(i), TSEncoding.PLAIN));
@@ -290,7 +290,7 @@ public class TsFileGeneratorUtils {
TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(pageSize);
try (TsFileWriter tsFileWriter = new TsFileWriter(file)) {
// register nonAlign timeseries
- List<MeasurementSchema> measurementSchemas = new ArrayList<>();
+ List<IMeasurementSchema> measurementSchemas = new ArrayList<>();
for (int i = 0; i < measurementNum; i++) {
measurementSchemas.add(new MeasurementSchema("s" + i, getDataType(i), TSEncoding.PLAIN));
}
@@ -334,7 +334,7 @@ public class TsFileGeneratorUtils {
TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(pageSize);
try (TsFileWriter tsFileWriter = new TsFileWriter(file)) {
// register align timeseries
- List<MeasurementSchema> alignedMeasurementSchemas = new ArrayList<>();
+ List<IMeasurementSchema> alignedMeasurementSchemas = new ArrayList<>();
for (int i = 0; i < measurementIndex.size(); i++) {
alignedMeasurementSchemas.add(
new MeasurementSchema(
@@ -392,7 +392,7 @@ public class TsFileGeneratorUtils {
TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(pageSize);
try (TsFileWriter tsFileWriter = new TsFileWriter(file)) {
// register nonAlign timeseries
- List<MeasurementSchema> measurementSchemas = new ArrayList<>();
+ List<IMeasurementSchema> measurementSchemas = new ArrayList<>();
for (int i = 0; i < measurementIndex.size(); i++) {
measurementSchemas.add(
new MeasurementSchema(
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileSketchTool.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileSketchTool.java
new file mode 100644
index 00000000..e7ce355a
--- /dev/null
+++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileSketchTool.java
@@ -0,0 +1,730 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tsfile.utils;
+
+import org.apache.tsfile.common.conf.TSFileConfig;
+import org.apache.tsfile.file.IMetadataIndexEntry;
+import org.apache.tsfile.file.MetaMarker;
+import org.apache.tsfile.file.header.ChunkGroupHeader;
+import org.apache.tsfile.file.header.PageHeader;
+import org.apache.tsfile.file.metadata.ChunkGroupMetadata;
+import org.apache.tsfile.file.metadata.ChunkMetadata;
+import org.apache.tsfile.file.metadata.DeviceMetadataIndexEntry;
+import org.apache.tsfile.file.metadata.IDeviceID;
+import org.apache.tsfile.file.metadata.MetadataIndexNode;
+import org.apache.tsfile.file.metadata.TableSchema;
+import org.apache.tsfile.file.metadata.TimeseriesMetadata;
+import org.apache.tsfile.file.metadata.TsFileMetadata;
+import org.apache.tsfile.file.metadata.enums.MetadataIndexNodeType;
+import org.apache.tsfile.fileSystem.FSFactoryProducer;
+import org.apache.tsfile.read.TsFileCheckStatus;
+import org.apache.tsfile.read.TsFileSequenceReader;
+import org.apache.tsfile.read.common.Chunk;
+import org.apache.tsfile.read.common.Path;
+import org.apache.tsfile.write.schema.Schema;
+
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.nio.BufferOverflowException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.TreeMap;
+
+public class TsFileSketchTool {
+
+ private String filename;
+ private PrintWriter pw;
+ private TsFileSketchToolReader reader;
+ private String splitStr; // for split different part of TsFile
+ TsFileMetadata tsFileMetaData;
+ List<ChunkGroupMetadata> allChunkGroupMetadata;
+
+ public static void main(String[] args) throws IOException {
+ Pair<String, String> fileNames = checkArgs(args);
+ String filename = fileNames.left;
+ String outFile = fileNames.right;
+ System.out.println("TsFile path:" + filename);
+ System.out.println("Sketch save path:" + outFile);
+ new TsFileSketchTool(filename, outFile).run();
+ }
+
+ /**
+ * construct TsFileSketchTool
+ *
+ * @param filename input file path
+ * @param outFile output file path
+ */
+ public TsFileSketchTool(String filename, String outFile) {
+ try {
+ this.filename = filename;
+ pw = new PrintWriter(new FileWriter(outFile));
+ reader = new TsFileSketchToolReader(filename);
+ StringBuilder str1 = new StringBuilder();
+ for (int i = 0; i < 21; i++) {
+ str1.append("|");
+ }
+ splitStr = str1.toString();
+ // get metadata information
+ tsFileMetaData = reader.readFileMetadata();
+ allChunkGroupMetadata = new ArrayList<>();
+ if (reader.selfCheck(new Schema(), allChunkGroupMetadata, false)
+ != TsFileCheckStatus.COMPLETE_FILE) {
+ throw new IOException(
+ String.format("Cannot load file %s because the file has crashed.", filename));
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ /** entry of tool */
+ public void run() throws IOException {
+ long length = FSFactoryProducer.getFSFactory().getFile(filename).length();
+ printlnBoth(
+ pw, "-------------------------------- TsFile Sketch --------------------------------");
+ printlnBoth(pw, "file path: " + filename);
+ printlnBoth(pw, "file length: " + length);
+
+ // print file information
+ printFileInfo();
+
+ // print chunk
+ printChunk(allChunkGroupMetadata);
+
+ // metadata begins
+ if (tsFileMetaData.getTableMetadataIndexNodeMap().isEmpty()) {
+ printlnBoth(
+ pw, String.format("%20s", reader.getFileMetadataPos() - 1) + "|\t" + "[marker] 2");
+ } else {
+ printlnBoth(
+ pw,
+ String.format("%20s", reader.readFileMetadata().getMetaOffset()) + "|\t" + "[marker] 2");
+ }
+
+ // get all TimerseriesIndex
+ Map<Long, Pair<Path, TimeseriesMetadata>> timeseriesMetadataMap =
+ reader.getAllTimeseriesMetadataWithOffset();
+
+ // get all IndexOfTimerseriesIndex (excluding the root node in TsFileMetadata)
+ TreeMap<Long, MetadataIndexNode> metadataIndexNodeMap = new TreeMap<>();
+ List<String> treeOutputStringBuffer = new ArrayList<>();
+ for (Entry<String, MetadataIndexNode> entry :
+ tsFileMetaData.getTableMetadataIndexNodeMap().entrySet()) {
+ treeOutputStringBuffer.add(entry.getKey());
+ loadIndexTree(entry.getValue(), metadataIndexNodeMap, treeOutputStringBuffer, 0);
+ }
+
+ // iterate timeseriesMetadataMap and metadataIndexNodeMap to print info in increasing order of
+ // position
+ Iterator<Entry<Long, Pair<Path, TimeseriesMetadata>>> ite1 =
+ timeseriesMetadataMap.entrySet().iterator();
+ Iterator<Entry<Long, MetadataIndexNode>> ite2 = metadataIndexNodeMap.entrySet().iterator();
+ Entry<Long, Pair<Path, TimeseriesMetadata>> value1 = (ite1.hasNext() ? ite1.next() : null);
+ Entry<Long, MetadataIndexNode> value2 = (ite2.hasNext() ? ite2.next() : null);
+ while (value1 != null || value2 != null) {
+ if (value2 == null || (value1 != null && value1.getKey().compareTo(value2.getKey()) <= 0)) {
+ printTimeseriesIndex(value1.getKey(), value1.getValue());
+ value1 = (ite1.hasNext() ? ite1.next() : null);
+ } else {
+ printIndexOfTimerseriesIndex(value2.getKey(), value2.getValue());
+ value2 = (ite2.hasNext() ? ite2.next() : null);
+ }
+ }
+
+ // print TsFile Metadata
+ printTsFileMetadata(tsFileMetaData);
+
+ printlnBoth(pw, String.format("%20s", length) + "|\tEND of TsFile");
+ printlnBoth(
+ pw,
+ "---------------------------- IndexOfTimerseriesIndex Tree -----------------------------");
+ // print index tree
+ for (String str : treeOutputStringBuffer) {
+ printlnBoth(pw, str);
+ }
+ printlnBoth(
+ pw,
+ "---------------------------------- TsFile Sketch End ----------------------------------");
+ reader.close();
+ pw.close();
+ }
+
+ public void close() throws IOException {
+ reader.close();
+ pw.close();
+ }
+
+ private void printTsFileMetadata(TsFileMetadata tsFileMetaData) {
+ try {
+ long pos = reader.getFileMetadataPos();
+ printlnBoth(pw, splitStr + " [TsFileMetadata] begins");
+
+ // metadataIndex
+ printlnBoth(
+ pw,
+ String.format("%20s", pos)
+ + "|\tTableIndexRootCnt="
+ + tsFileMetaData.getTableMetadataIndexNodeMap().size());
+ pos += Integer.BYTES;
+ for (Entry<String, MetadataIndexNode> entry :
+ tsFileMetaData.getTableMetadataIndexNodeMap().entrySet()) {
+ printlnBoth(
+ pw,
+ String.format("%20s", pos)
+ + "|\t[Table Name] "
+ + entry.getKey()
+ + ", size="
+ + ReadWriteForEncodingUtils.intStringSize(entry.getKey()));
+ pos += ReadWriteForEncodingUtils.intStringSize(entry.getKey());
+ pos = printIndexOfTimerseriesIndex(pos, entry.getValue());
+ }
+
+ // table schema
+ printlnBoth(
+ pw,
+ String.format("%20s", pos)
+ + "|\tTableSchemaCnt="
+ + tsFileMetaData.getTableSchemaMap().size());
+ pos += Integer.BYTES;
+ for (Entry<String, TableSchema> entry : tsFileMetaData.getTableSchemaMap().entrySet()) {
+ final String tableName = entry.getKey();
+ final TableSchema tableSchema = entry.getValue();
+
+ final int serializedSize = tableSchema.serializedSize();
+ printlnBoth(
+ pw,
+ String.format("%20s", pos)
+ + "|\t[TableSchema] "
+ + tableSchema
+ + ", size="
+ + serializedSize);
+ pos += ReadWriteForEncodingUtils.intStringSize(tableName) + serializedSize;
+ }
+
+ // metaOffset
+ printlnBoth(
+ pw, String.format("%20s", pos) + "|\t[Meta Offset] " + tsFileMetaData.getMetaOffset());
+ pos += Long.BYTES;
+
+ // bloom filter
+
+ BloomFilter bloomFilter = tsFileMetaData.getBloomFilter();
+ if (bloomFilter != null) {
+ final int length = bloomFilter.serialize().length;
+ printlnBoth(
+ pw,
+ String.format("%20s", pos)
+ + "|\t[Bloom Filter Size] "
+ + "bit vector byte array length="
+ + length
+ + bloomFilter.getHashFunctionSize());
+ pos += Integer.BYTES;
+ printlnBoth(
+ pw,
+ String.format("%20s", pos)
+ + "|\t[Bloom Filter] "
+ + ", filterCapacity="
+ + bloomFilter.getSize()
+ + ", hashFunctionSize="
+ + bloomFilter.getHashFunctionSize());
+ pos += length;
+ }
+
+ printlnBoth(pw, splitStr + " [TsFileMetadata] ends");
+
+ printlnBoth(
+ pw,
+ String.format("%20s", (reader.getFileMetadataPos() + reader.getTsFileMetadataSize()))
+ + "|\t[TsFileMetadataSize] "
+ + reader.getTsFileMetadataSize());
+
+ printlnBoth(
+ pw,
+ String.format("%20s", reader.getFileMetadataPos() + reader.getTsFileMetadataSize() + 4)
+ + "|\t[magic tail] "
+ + reader.readTailMagic());
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ private long printIndexOfTimerseriesIndex(long pos, MetadataIndexNode metadataIndexNode) {
+ printlnBoth(pw, String.format("%20s", pos) + "|\t[MetadataIndexNode]");
+ printlnBoth(
+ pw,
+ String.format("%20s", pos) + "|\t\t childrenCnt=" + metadataIndexNode.getChildren().size());
+ pos += ReadWriteForEncodingUtils.uVarIntSize(metadataIndexNode.getChildren().size());
+ for (IMetadataIndexEntry metadataIndexEntry : metadataIndexNode.getChildren()) {
+ printlnBoth(
+ pw,
+ String.format("%20s", pos)
+ + "|\t\t<"
+ + metadataIndexEntry.getCompareKey()
+ + ", "
+ + metadataIndexEntry.getOffset()
+ + ">");
+ pos += metadataIndexEntry.serializedSize();
+ }
+ printlnBoth(
+ pw, String.format("%20s", pos) + "|\t\tendOffset=" + metadataIndexNode.getEndOffset());
+ pos += Long.BYTES;
+ printlnBoth(
+ pw, String.format("%20s", pos) + "|\t\tnodeType=" + metadataIndexNode.getNodeType());
+ pos += Byte.BYTES;
+ return pos;
+ }
+
+ private void printFileInfo() {
+ try {
+ printlnBoth(pw, "");
+ printlnBoth(pw, String.format("%20s", "POSITION") + "|\tCONTENT");
+ printlnBoth(pw, String.format("%20s", "--------") + " \t-------");
+ printlnBoth(pw, String.format("%20d", 0) + "|\t[magic head] " + reader.readHeadMagic());
+ printlnBoth(
+ pw,
+ String.format("%20d", TSFileConfig.MAGIC_STRING.getBytes().length)
+ + "|\t[version number] "
+ + reader.readVersionNumber());
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ private void printChunk(List<ChunkGroupMetadata> allChunkGroupMetadata) {
+ try {
+ long nextChunkGroupHeaderPos =
+ (long) TSFileConfig.MAGIC_STRING.getBytes().length + Byte.BYTES;
+ // ChunkGroup begins
+ for (ChunkGroupMetadata chunkGroupMetadata : allChunkGroupMetadata) {
+ printlnBoth(
+ pw,
+ splitStr
+ + " [ChunkGroup] of "
+ + chunkGroupMetadata.getDevice()
+ + ", num of Chunks:"
+ + chunkGroupMetadata.getChunkMetadataList().size());
+ // chunkGroupHeader begins
+ long offset = nextChunkGroupHeaderPos;
+ printlnBoth(pw, String.format("%20s", offset) + "|\t[ChunkGroup Header]");
+ ChunkGroupHeader chunkGroupHeader =
+ reader.readChunkGroupHeader(nextChunkGroupHeaderPos, false);
+ printlnBoth(pw, String.format("%20s", offset) + "|\t\t[marker] 0");
+ printlnBoth(
+ pw,
+ String.format("%20s", offset + 1)
+ + "|\t\t[deviceID] "
+ + chunkGroupHeader.getDeviceID()
+ + " size="
+ + chunkGroupHeader.getDeviceID().serializedSize());
+ // chunk begins
+ for (ChunkMetadata chunkMetadata : chunkGroupMetadata.getChunkMetadataList()) {
+ offset = chunkMetadata.getOffsetOfChunkHeader();
+ Chunk chunk = reader.readMemChunk(chunkMetadata);
+ printlnBoth(
+ pw,
+ String.format("%20d", offset)
+ + "|\t[Chunk] of "
+ + new Path(
+ chunkGroupHeader.getDeviceID(), chunkMetadata.getMeasurementUid(), false)
+ + ", "
+ + chunkMetadata.getStatistics());
+ printlnBoth(
+ pw,
+ String.format("%20s", offset)
+ + "|\t\t[Chunk Header] "
+ + "marker="
+ + chunk.getHeader().getChunkType()
+ + ", measurementID="
+ + chunk.getHeader().getMeasurementID()
+ + ", dataSize="
+ + chunk.getHeader().getDataSize()
+ + ", dataType="
+ + chunk.getHeader().getDataType()
+ + ", compressionType="
+ + chunk.getHeader().getCompressionType()
+ + ", encodingType="
+ + chunk.getHeader().getEncodingType()
+ + ", size="
+ + chunk.getHeader().getSerializedSize());
+ offset += chunk.getHeader().getSerializedSize();
+ PageHeader pageHeader;
+ if (((byte) (chunk.getHeader().getChunkType() & 0x3F))
+ == MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER) {
+ pageHeader = PageHeader.deserializeFrom(chunk.getData(), chunkMetadata.getStatistics());
+ printlnBoth(
+ pw,
+ String.format("%20s", offset)
+ + "|\t\t\t[Page Header] "
+ + " HeaderSize:"
+ + (pageHeader.getSerializedSize()
+ - pageHeader.getStatistics().getSerializedSize())
+ + ", UncompressedSize:"
+ + pageHeader.getUncompressedSize()
+ + ", CompressedSize:"
+ + pageHeader.getCompressedSize());
+ offset +=
+ pageHeader.getSerializedSize() - pageHeader.getStatistics().getSerializedSize();
+ printlnBoth(
+ pw,
+ String.format("%20s", offset)
+ + "|\t\t\t[Page Data] "
+ + " Size:"
+ + pageHeader.getCompressedSize());
+ } else { // more than one page in this chunk
+ ByteBuffer chunkDataBuffer = chunk.getData();
+
+ int pageID = 0;
+ while (chunkDataBuffer.remaining() > 0) {
+ pageID++;
+ // deserialize a PageHeader from chunkDataBuffer
+ pageHeader =
+ PageHeader.deserializeFrom(chunkDataBuffer, chunk.getHeader().getDataType());
+ // skip the compressed bytes
+ chunkDataBuffer.position(chunkDataBuffer.position() + pageHeader.getCompressedSize());
+ // print page info
+ printlnBoth(
+ pw,
+ String.format("%20s", offset)
+ + String.format("|\t\t\t[PageHeader-%s] ", pageID)
+ + " HeaderSize:"
+ + pageHeader.getSerializedSize()
+ + ", UncompressedSize:"
+ + pageHeader.getUncompressedSize()
+ + ", CompressedSize:"
+ + pageHeader.getCompressedSize()
+ + ", "
+ + pageHeader.getStatistics());
+ offset += pageHeader.getSerializedSize();
+ printlnBoth(
+ pw,
+ String.format("%20s", offset)
+ + String.format("|\t\t\t[Page-%s] ", pageID)
+ + ", CompressedSize:"
+ + pageHeader.getCompressedSize()
+ + ", "
+ + pageHeader.getStatistics());
+ offset += pageHeader.getCompressedSize();
+ }
+ }
+ nextChunkGroupHeaderPos =
+ chunkMetadata.getOffsetOfChunkHeader()
+ + chunk.getHeader().getSerializedSize()
+ + chunk.getHeader().getDataSize();
+ }
+ reader.position(nextChunkGroupHeaderPos);
+ byte marker = reader.readMarker();
+ switch (marker) {
+ case MetaMarker.CHUNK_GROUP_HEADER:
+ // do nothing
+ break;
+ case MetaMarker.OPERATION_INDEX_RANGE:
+ // skip the PlanIndex
+ nextChunkGroupHeaderPos += 16;
+ break;
+ }
+
+ printlnBoth(pw, splitStr + " [ChunkGroup] of " + chunkGroupMetadata.getDevice() + " ends");
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ private void printTimeseriesIndex(long pos, Pair<Path, TimeseriesMetadata> timeseriesMetadata) {
+ try {
+ printlnBoth(
+ pw,
+ String.format("%20s", pos)
+ + "|\t[TimeseriesMetadata] of "
+ + timeseriesMetadata.left
+ + ", tsDataType:"
+ + timeseriesMetadata.right.getTsDataType()
+ + ", sizeWithoutChunkMetadata:"
+ + timeseriesMetadata.right.serializedSizeWithoutMetadata()
+ + ", "
+ + timeseriesMetadata.right.getStatistics());
+ pos += timeseriesMetadata.getRight().serializedSizeWithoutMetadata();
+ List<ChunkMetadata> chunkMetadataList = reader.getChunkMetadataList(timeseriesMetadata.left);
+ for (int i = 0; i < chunkMetadataList.size(); i++) {
+ ChunkMetadata chunkMetadata = chunkMetadataList.get(i);
+ final int serializedSize = chunkMetadata.serializedSize(i != 0);
+ printlnBoth(
+ pw,
+ String.format("%20s", pos)
+ + "|\t\t[ChunkMetadata] "
+ + "offset="
+ + chunkMetadata.getOffsetOfChunkHeader()
+ + ", size="
+ + serializedSize);
+ pos += serializedSize;
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ /**
+ * load by dfs, and sort by TreeMap
+ *
+ * @param metadataIndexNode current node
+ * @param metadataIndexNodeMap result map, key is offset
+ * @param treeOutputStringBuffer result list, string is index tree
+ * @param deep current deep
+ */
+ private void loadIndexTree(
+ MetadataIndexNode metadataIndexNode,
+ TreeMap<Long, MetadataIndexNode> metadataIndexNodeMap,
+ List<String> treeOutputStringBuffer,
+ int deep)
+ throws IOException {
+ StringBuilder tableWriter = new StringBuilder("\t");
+ for (int i = 0; i < deep; i++) {
+ tableWriter.append("\t\t");
+ }
+ treeOutputStringBuffer.add(
+ tableWriter + "[MetadataIndex:" + metadataIndexNode.getNodeType() + "]");
+ for (int i = 0; i < metadataIndexNode.getChildren().size(); i++) {
+ IMetadataIndexEntry metadataIndexEntry = metadataIndexNode.getChildren().get(i);
+
+ treeOutputStringBuffer.add(
+ tableWriter
+ + "└──────["
+ + metadataIndexEntry.getCompareKey()
+ + ","
+ + metadataIndexEntry.getOffset()
+ + "]");
+ if (!metadataIndexNode.getNodeType().equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) {
+ long endOffset = metadataIndexNode.getEndOffset();
+ if (i != metadataIndexNode.getChildren().size() - 1) {
+ endOffset = metadataIndexNode.getChildren().get(i + 1).getOffset();
+ }
+ boolean currentChildLevelIsDevice =
+ MetadataIndexNodeType.INTERNAL_DEVICE.equals(metadataIndexNode.getNodeType());
+ MetadataIndexNode subNode =
+ reader.readMetadataIndexNode(
+ metadataIndexEntry.getOffset(), endOffset, currentChildLevelIsDevice);
+ metadataIndexNodeMap.put(metadataIndexEntry.getOffset(), subNode);
+ loadIndexTree(subNode, metadataIndexNodeMap, treeOutputStringBuffer, deep + 1);
+ }
+ }
+ }
+
+ private void printlnBoth(PrintWriter pw, String str) {
+ System.out.println(str);
+ pw.println(str);
+ }
+
+ private static Pair<String, String> checkArgs(String[] args) {
+ String filename = "test.tsfile";
+ String outFile = "TsFile_sketch_view.txt";
+ if (args.length == 1) {
+ filename = args[0];
+ } else if (args.length == 2) {
+ filename = args[0];
+ outFile = args[1];
+ }
+ return new Pair<>(filename, outFile);
+ }
+
+ private class TsFileSketchToolReader extends TsFileSequenceReader {
+
+ public TsFileSketchToolReader(String file) throws IOException {
+ super(file);
+ }
+
+ /**
+ * Traverse the metadata index from MetadataIndexEntry to get TimeseriesMetadatas
+ *
+ * @param metadataIndex MetadataIndexEntry
+ * @param buffer byte buffer
+ * @param deviceId String
+ * @param timeseriesMetadataMap map: deviceId -> timeseriesMetadata list
+ * @param needChunkMetadata deserialize chunk metadata list or not
+ */
+ private void generateMetadataIndexWithOffset(
+ long startOffset,
+ IMetadataIndexEntry metadataIndex,
+ ByteBuffer buffer,
+ IDeviceID deviceId,
+ MetadataIndexNodeType type,
+ Map<Long, Pair<Path, TimeseriesMetadata>> timeseriesMetadataMap,
+ boolean needChunkMetadata)
+ throws IOException {
+ try {
+ if (type.equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) {
+ while (buffer.hasRemaining()) {
+ long pos = startOffset + buffer.position();
+ TimeseriesMetadata timeseriesMetadata =
+ TimeseriesMetadata.deserializeFrom(buffer, needChunkMetadata);
+ timeseriesMetadataMap.put(
+ pos,
+ new Pair<>(
+ new Path(deviceId, timeseriesMetadata.getMeasurementId(), true),
+ timeseriesMetadata));
+ }
+ } else {
+ // deviceId should be determined by LEAF_DEVICE node
+ if (type.equals(MetadataIndexNodeType.LEAF_DEVICE)) {
+ deviceId = ((DeviceMetadataIndexEntry) metadataIndex).getDeviceID();
+ }
+ boolean currentChildLevelIsDevice = MetadataIndexNodeType.INTERNAL_DEVICE.equals(type);
+ MetadataIndexNode metadataIndexNode =
+ reader
+ .getDeserializeContext()
+ .deserializeMetadataIndexNode(buffer, currentChildLevelIsDevice);
+ int metadataIndexListSize = metadataIndexNode.getChildren().size();
+ for (int i = 0; i < metadataIndexListSize; i++) {
+ long endOffset = metadataIndexNode.getEndOffset();
+ if (i != metadataIndexListSize - 1) {
+ endOffset = metadataIndexNode.getChildren().get(i + 1).getOffset();
+ }
+ if (endOffset - metadataIndexNode.getChildren().get(i).getOffset()
+ < Integer.MAX_VALUE) {
+ ByteBuffer nextBuffer =
+ readData(metadataIndexNode.getChildren().get(i).getOffset(), endOffset);
+ generateMetadataIndexWithOffset(
+ metadataIndexNode.getChildren().get(i).getOffset(),
+ metadataIndexNode.getChildren().get(i),
+ nextBuffer,
+ deviceId,
+ metadataIndexNode.getNodeType(),
+ timeseriesMetadataMap,
+ needChunkMetadata);
+ } else {
+ // when the buffer length is over than Integer.MAX_VALUE,
+ // using tsFileInput to get timeseriesMetadataList
+ generateMetadataIndexWithOffsetUsingTsFileInput(
+ metadataIndexNode.getChildren().get(i).getOffset(),
+ endOffset,
+ metadataIndexNode.getChildren().get(i),
+ deviceId,
+ metadataIndexNode.getNodeType(),
+ timeseriesMetadataMap,
+ needChunkMetadata);
+ }
+ }
+ }
+ } catch (BufferOverflowException e) {
+ throw e;
+ }
+ }
+
+ /**
+ * Traverse the metadata index from MetadataIndexEntry to get TimeseriesMetadatas
+ *
+ * @param metadataIndex MetadataIndexEntry
+ * @param deviceId String
+ * @param timeseriesMetadataMap map: deviceId -> timeseriesMetadata list
+ * @param needChunkMetadata deserialize chunk metadata list or not
+ */
+ private void generateMetadataIndexWithOffsetUsingTsFileInput(
+ long start,
+ long end,
+ IMetadataIndexEntry metadataIndex,
+ IDeviceID deviceId,
+ MetadataIndexNodeType type,
+ Map<Long, Pair<Path, TimeseriesMetadata>> timeseriesMetadataMap,
+ boolean needChunkMetadata)
+ throws IOException {
+ try {
+ tsFileInput.position(start);
+ if (type.equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) {
+ while (tsFileInput.position() < end) {
+ long pos = tsFileInput.position();
+ TimeseriesMetadata timeseriesMetadata =
+ TimeseriesMetadata.deserializeFrom(tsFileInput, needChunkMetadata);
+ timeseriesMetadataMap.put(
+ pos,
+ new Pair<>(
+ new Path(deviceId, timeseriesMetadata.getMeasurementId(), true),
+ timeseriesMetadata));
+ }
+ } else {
+ // deviceId should be determined by LEAF_DEVICE node
+ if (type.equals(MetadataIndexNodeType.LEAF_DEVICE)) {
+ deviceId = ((DeviceMetadataIndexEntry) metadataIndex).getDeviceID();
+ }
+ boolean isDeviceLevel = MetadataIndexNodeType.INTERNAL_DEVICE.equals(type);
+ MetadataIndexNode metadataIndexNode =
+ reader
+ .getDeserializeContext()
+ .deserializeMetadataIndexNode(tsFileInput.wrapAsInputStream(), isDeviceLevel);
+ int metadataIndexListSize = metadataIndexNode.getChildren().size();
+ for (int i = 0; i < metadataIndexListSize; i++) {
+ long endOffset = metadataIndexNode.getEndOffset();
+ if (i != metadataIndexListSize - 1) {
+ endOffset = metadataIndexNode.getChildren().get(i + 1).getOffset();
+ }
+ generateMetadataIndexWithOffsetUsingTsFileInput(
+ metadataIndexNode.getChildren().get(i).getOffset(),
+ endOffset,
+ metadataIndexNode.getChildren().get(i),
+ deviceId,
+ metadataIndexNode.getNodeType(),
+ timeseriesMetadataMap,
+ needChunkMetadata);
+ }
+ }
+ } catch (BufferOverflowException e) {
+ throw e;
+ }
+ }
+
+ public Map<Long, Pair<Path, TimeseriesMetadata>> getAllTimeseriesMetadataWithOffset()
+ throws IOException {
+ if (tsFileMetaData == null) {
+ readFileMetadata();
+ }
+ Map<Long, Pair<Path, TimeseriesMetadata>> timeseriesMetadataMap = new TreeMap<>();
+ for (Entry<String, MetadataIndexNode> entry :
+ tsFileMetaData.getTableMetadataIndexNodeMap().entrySet()) {
+ List<IMetadataIndexEntry> metadataIndexEntryList = entry.getValue().getChildren();
+ for (int i = 0; i < metadataIndexEntryList.size(); i++) {
+ IMetadataIndexEntry metadataIndexEntry = metadataIndexEntryList.get(i);
+ long endOffset = entry.getValue().getEndOffset();
+ if (i != metadataIndexEntryList.size() - 1) {
+ endOffset = metadataIndexEntryList.get(i + 1).getOffset();
+ }
+ ByteBuffer buffer = readData(metadataIndexEntry.getOffset(), endOffset);
+ generateMetadataIndexWithOffset(
+ metadataIndexEntry.getOffset(),
+ metadataIndexEntry,
+ buffer,
+ null,
+ entry.getValue().getNodeType(),
+ timeseriesMetadataMap,
+ false);
+ }
+ }
+
+ return timeseriesMetadataMap;
+ }
+ }
+
+ // for test
+ protected List<ChunkGroupMetadata> getAllChunkGroupMetadata() {
+ return allChunkGroupMetadata;
+ }
+}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/WriteUtils.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/WriteUtils.java
new file mode 100644
index 00000000..542d252f
--- /dev/null
+++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/WriteUtils.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tsfile.utils;
+
+import org.apache.tsfile.file.metadata.IDeviceID;
+import org.apache.tsfile.write.record.Tablet;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class WriteUtils {
+
+ /**
+ * A Tablet for the table-view insert interface may contain more than one device. This method
+ * splits a Tablet by different deviceIds so that the caller can insert them device-by-device.
+ *
+ * @return (deviceId, endRowNum) pairs
+ */
+ public static List<Pair<IDeviceID, Integer>> splitTabletByDevice(Tablet tablet) {
+ List<Pair<IDeviceID, Integer>> result = new ArrayList<>();
+ IDeviceID lastDeviceID = null;
+ for (int i = 0; i < tablet.rowSize; i++) {
+ final IDeviceID currDeviceID = tablet.getDeviceID(i);
+ if (!currDeviceID.equals(lastDeviceID)) {
+ if (lastDeviceID != null) {
+ result.add(new Pair<>(lastDeviceID, i));
+ }
+ lastDeviceID = currDeviceID;
+ }
+ }
+ result.add(new Pair<>(lastDeviceID, tablet.rowSize));
+ return result;
+ }
+
+ public static int compareStrings(String a, String b) {
+ if (a == null && b == null) {
+ return 0;
+ }
+ if (a == null) {
+ return -1;
+ }
+ if (b == null) {
+ return 1;
+ }
+ return a.compareTo(b);
+ }
+}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/TsFileWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/TsFileWriter.java
index f7c2f293..f3cc2036 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/write/TsFileWriter.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/write/TsFileWriter.java
@@ -20,12 +20,17 @@ package org.apache.tsfile.write;
import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.common.conf.TSFileDescriptor;
+import org.apache.tsfile.exception.write.ConflictDataTypeException;
+import org.apache.tsfile.exception.write.NoDeviceException;
import org.apache.tsfile.exception.write.NoMeasurementException;
+import org.apache.tsfile.exception.write.NoTableException;
import org.apache.tsfile.exception.write.WriteProcessException;
import org.apache.tsfile.file.metadata.IDeviceID;
-import org.apache.tsfile.file.metadata.PlainDeviceID;
+import org.apache.tsfile.file.metadata.TableSchema;
import org.apache.tsfile.read.common.Path;
import org.apache.tsfile.utils.MeasurementGroup;
+import org.apache.tsfile.utils.Pair;
+import org.apache.tsfile.utils.WriteUtils;
import org.apache.tsfile.write.chunk.AlignedChunkGroupWriterImpl;
import org.apache.tsfile.write.chunk.IChunkGroupWriter;
import org.apache.tsfile.write.chunk.NonAlignedChunkGroupWriterImpl;
@@ -33,9 +38,7 @@ import org.apache.tsfile.write.record.TSRecord;
import org.apache.tsfile.write.record.Tablet;
import org.apache.tsfile.write.record.datapoint.DataPoint;
import org.apache.tsfile.write.schema.IMeasurementSchema;
-import org.apache.tsfile.write.schema.MeasurementSchema;
import org.apache.tsfile.write.schema.Schema;
-import org.apache.tsfile.write.schema.VectorMeasurementSchema;
import org.apache.tsfile.write.writer.RestorableTsFileIOWriter;
import org.apache.tsfile.write.writer.TsFileIOWriter;
import org.apache.tsfile.write.writer.TsFileOutput;
@@ -49,6 +52,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.TreeMap;
/**
* TsFileWriter is the entrance for writing processing. It receives a record and send it to
@@ -62,9 +66,6 @@ public class TsFileWriter implements AutoCloseable {
protected static final TSFileConfig config = TSFileDescriptor.getInstance().getConfig();
private static final Logger LOG = LoggerFactory.getLogger(TsFileWriter.class);
- /** schema of this TsFile. */
- protected final Schema schema;
-
/** IO writer of this TsFile. */
private final TsFileIOWriter fileWriter;
@@ -86,13 +87,15 @@ public class TsFileWriter implements AutoCloseable {
*/
private boolean isUnseq = false;
- private Map<IDeviceID, IChunkGroupWriter> groupWriters = new HashMap<>();
+ private Map<IDeviceID, IChunkGroupWriter> groupWriters = new TreeMap<>();
/** min value of threshold of data points num check. */
private long recordCountForNextMemCheck = 100;
private long chunkGroupSizeThreshold;
+ private boolean isTableWriteAligned = true;
+
/**
* init this TsFileWriter.
*
@@ -126,7 +129,6 @@ public class TsFileWriter implements AutoCloseable {
*
* @param output the TsFileOutput of the file to be written by this TsFileWriter
* @param schema the schema of this TsFile
- * @throws IOException
*/
public TsFileWriter(TsFileOutput output, Schema schema) throws IOException {
this(new TsFileIOWriter(output), schema, TSFileDescriptor.getInstance().getConfig());
@@ -159,41 +161,10 @@ public class TsFileWriter implements AutoCloseable {
this.fileWriter = fileWriter;
if (fileWriter instanceof RestorableTsFileIOWriter) {
- Map<Path, IMeasurementSchema> schemaMap =
- ((RestorableTsFileIOWriter) fileWriter).getKnownSchema();
- Map<Path, MeasurementGroup> measurementGroupMap = new HashMap<>();
- for (Map.Entry<Path, IMeasurementSchema> entry : schemaMap.entrySet()) {
- IMeasurementSchema measurementSchema = entry.getValue();
- if (measurementSchema instanceof VectorMeasurementSchema) {
- MeasurementGroup group =
- measurementGroupMap.getOrDefault(
- new Path(entry.getKey().getDevice()), new MeasurementGroup(true));
- List<String> measurementList = measurementSchema.getSubMeasurementsList();
- for (int i = 0; i < measurementList.size(); i++) {
- group
- .getMeasurementSchemaMap()
- .put(
- measurementList.get(i),
- new MeasurementSchema(
- measurementList.get(i),
- measurementSchema.getSubMeasurementsTSDataTypeList().get(i),
- measurementSchema.getSubMeasurementsTSEncodingList().get(i)));
- }
- measurementGroupMap.put(new Path(entry.getKey().getDevice()), group);
- } else {
- MeasurementGroup group =
- measurementGroupMap.getOrDefault(
- new Path(entry.getKey().getDevice()), new MeasurementGroup(false));
- group
- .getMeasurementSchemaMap()
- .put(measurementSchema.getMeasurementId(), (MeasurementSchema) measurementSchema);
- measurementGroupMap.put(new Path(entry.getKey().getDevice()), group);
... 4384 lines suppressed ...
(tsfile) 04/13: Bump org.apache.commons:commons-lang3 from 3.13.0 to 3.15.0 (#187)
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch iotdb
in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit 12c37a17a887e27e3c0bcb68e06c73af3546c496
Author: dependabot[bot] <49...@users.noreply.github.com>
AuthorDate: Thu Aug 1 15:02:10 2024 +0800
Bump org.apache.commons:commons-lang3 from 3.13.0 to 3.15.0 (#187)
Bumps org.apache.commons:commons-lang3 from 3.13.0 to 3.15.0.
---
updated-dependencies:
- dependency-name: org.apache.commons:commons-lang3
dependency-type: direct:production
update-type: version-update:semver-minor
...
Signed-off-by: dependabot[bot] <su...@github.com>
Co-authored-by: dependabot[bot] <49...@users.noreply.github.com>
---
java/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/java/pom.xml b/java/pom.xml
index 99b0565a..fe5ed453 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -55,7 +55,7 @@
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
- <version>3.13.0</version>
+ <version>3.15.0</version>
</dependency>
<dependency>
<groupId>org.lz4</groupId>
(tsfile) 01/13: Update Powered-By.md (#45)
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch iotdb
in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit 1210c1167e011d4d961981ec7617537433e5ada1
Author: majialin <10...@users.noreply.github.com>
AuthorDate: Tue Jul 30 10:09:19 2024 +0800
Update Powered-By.md (#45)
---
docs/src/zh/Development/Powered-By.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docs/src/zh/Development/Powered-By.md b/docs/src/zh/Development/Powered-By.md
index 829f1a16..41d4513d 100644
--- a/docs/src/zh/Development/Powered-By.md
+++ b/docs/src/zh/Development/Powered-By.md
@@ -19,7 +19,7 @@
-->
-## Powered-By
+# Powered-By
### 使用“TsFile”的项目和产品名称
(tsfile) 13/13: change version
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch iotdb
in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit af32f4987797aba611855402d749b783fe7b0f30
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Mon Aug 5 12:09:47 2024 +0800
change version
---
cpp/pom.xml | 2 +-
java/common/pom.xml | 2 +-
java/examples/pom.xml | 4 ++--
java/pom.xml | 4 ++--
java/tools/pom.xml | 6 +++---
java/tsfile/pom.xml | 4 ++--
pom.xml | 2 +-
python/pom.xml | 2 +-
8 files changed, 13 insertions(+), 13 deletions(-)
diff --git a/cpp/pom.xml b/cpp/pom.xml
index 65324616..1447dfe3 100644
--- a/cpp/pom.xml
+++ b/cpp/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.tsfile</groupId>
<artifactId>tsfile-parent</artifactId>
- <version>1.2.0-SNAPSHOT</version>
+ <version>1.2.0-8aaedb22-SNAPSHOT</version>
</parent>
<artifactId>tsfile-cpp</artifactId>
<packaging>pom</packaging>
diff --git a/java/common/pom.xml b/java/common/pom.xml
index fe503903..fbc65ca7 100644
--- a/java/common/pom.xml
+++ b/java/common/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.tsfile</groupId>
<artifactId>tsfile-java</artifactId>
- <version>1.2.0-SNAPSHOT</version>
+ <version>1.2.0-8aaedb22-SNAPSHOT</version>
</parent>
<artifactId>common</artifactId>
<name>TsFile: Java: Common</name>
diff --git a/java/examples/pom.xml b/java/examples/pom.xml
index b27fd5bf..4e54fd12 100644
--- a/java/examples/pom.xml
+++ b/java/examples/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.tsfile</groupId>
<artifactId>tsfile-java</artifactId>
- <version>1.2.0-SNAPSHOT</version>
+ <version>1.2.0-8aaedb22-SNAPSHOT</version>
</parent>
<artifactId>examples</artifactId>
<packaging>pom</packaging>
@@ -38,7 +38,7 @@
<dependency>
<groupId>org.apache.tsfile</groupId>
<artifactId>tsfile</artifactId>
- <version>1.2.0-SNAPSHOT</version>
+ <version>1.2.0-8aaedb22-SNAPSHOT</version>
</dependency>
</dependencies>
<build>
diff --git a/java/pom.xml b/java/pom.xml
index 24b2cb59..337901ea 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -24,10 +24,10 @@
<parent>
<groupId>org.apache.tsfile</groupId>
<artifactId>tsfile-parent</artifactId>
- <version>1.2.0-SNAPSHOT</version>
+ <version>1.2.0-8aaedb22-SNAPSHOT</version>
</parent>
<artifactId>tsfile-java</artifactId>
- <version>1.2.0-SNAPSHOT</version>
+ <version>1.2.0-8aaedb22-SNAPSHOT</version>
<packaging>pom</packaging>
<name>TsFile: Java</name>
<modules>
diff --git a/java/tools/pom.xml b/java/tools/pom.xml
index d46e695c..548168c3 100644
--- a/java/tools/pom.xml
+++ b/java/tools/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.tsfile</groupId>
<artifactId>tsfile-java</artifactId>
- <version>1.2.0-SNAPSHOT</version>
+ <version>1.2.0-8aaedb22-SNAPSHOT</version>
</parent>
<artifactId>tools</artifactId>
<name>TsFile: Java: Tools</name>
@@ -32,7 +32,7 @@
<dependency>
<groupId>org.apache.tsfile</groupId>
<artifactId>common</artifactId>
- <version>1.2.0-SNAPSHOT</version>
+ <version>1.2.0-8aaedb22-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>commons-cli</groupId>
@@ -46,7 +46,7 @@
<dependency>
<groupId>org.apache.tsfile</groupId>
<artifactId>tsfile</artifactId>
- <version>1.2.0-SNAPSHOT</version>
+ <version>1.2.0-8aaedb22-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
diff --git a/java/tsfile/pom.xml b/java/tsfile/pom.xml
index 8ab97881..d8f850a4 100644
--- a/java/tsfile/pom.xml
+++ b/java/tsfile/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.tsfile</groupId>
<artifactId>tsfile-java</artifactId>
- <version>1.2.0-SNAPSHOT</version>
+ <version>1.2.0-8aaedb22-SNAPSHOT</version>
</parent>
<artifactId>tsfile</artifactId>
<name>TsFile: Java: TsFile</name>
@@ -38,7 +38,7 @@
<dependency>
<groupId>org.apache.tsfile</groupId>
<artifactId>common</artifactId>
- <version>1.2.0-SNAPSHOT</version>
+ <version>1.2.0-8aaedb22-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.github.luben</groupId>
diff --git a/pom.xml b/pom.xml
index ed75b3c8..65a2648a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -28,7 +28,7 @@
</parent>
<groupId>org.apache.tsfile</groupId>
<artifactId>tsfile-parent</artifactId>
- <version>1.2.0-SNAPSHOT</version>
+ <version>1.2.0-8aaedb22-SNAPSHOT</version>
<packaging>pom</packaging>
<name>Apache TsFile Project Parent POM</name>
<properties>
diff --git a/python/pom.xml b/python/pom.xml
index 1a0180d8..75c5e48d 100644
--- a/python/pom.xml
+++ b/python/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.tsfile</groupId>
<artifactId>tsfile-parent</artifactId>
- <version>1.2.0-SNAPSHOT</version>
+ <version>1.2.0-8aaedb22-SNAPSHOT</version>
</parent>
<artifactId>tsfile-python</artifactId>
<packaging>pom</packaging>
(tsfile) 07/13: Bump com.gradle:common-custom-user-data-maven-extension from 1.12.4 to 2 (#188)
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch iotdb
in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit 5d63035a4abacd0fa37ee4f8c0b1d41fe13c0adf
Author: dependabot[bot] <49...@users.noreply.github.com>
AuthorDate: Thu Aug 1 21:17:36 2024 +0800
Bump com.gradle:common-custom-user-data-maven-extension from 1.12.4 to 2 (#188)
Bumps [com.gradle:common-custom-user-data-maven-extension](https://github.com/gradle/common-custom-user-data-maven-extension) from 1.12.4 to 2.
- [Release notes](https://github.com/gradle/common-custom-user-data-maven-extension/releases)
- [Commits](https://github.com/gradle/common-custom-user-data-maven-extension/compare/v1.12.4...v2)
---
updated-dependencies:
- dependency-name: com.gradle:common-custom-user-data-maven-extension
dependency-type: direct:production
update-type: version-update:semver-major
...
Signed-off-by: dependabot[bot] <su...@github.com>
Co-authored-by: dependabot[bot] <49...@users.noreply.github.com>
---
.mvn/extensions.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml
index ff89b130..177fc5d6 100644
--- a/.mvn/extensions.xml
+++ b/.mvn/extensions.xml
@@ -28,6 +28,6 @@
<extension>
<groupId>com.gradle</groupId>
<artifactId>common-custom-user-data-maven-extension</artifactId>
- <version>1.12.4</version>
+ <version>2</version>
</extension>
</extensions>
(tsfile) 02/13: Support codecov to show code coverage (#184)
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch iotdb
in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit 5acbbd0df48aef6009bd8102f0443f76b4108355
Author: Haonan <hh...@outlook.com>
AuthorDate: Wed Jul 31 14:25:36 2024 +0800
Support codecov to show code coverage (#184)
---
.github/workflows/code-coverage.yml | 35 ++++++
.github/workflows/unit-test.yml | 4 +-
README-zh.md | 3 +-
README.md | 1 +
code-coverage/copy-code-coverage-sources.sh | 31 +++++
code-coverage/pom.xml | 132 +++++++++++++++++++++
codecov.yml | 47 ++++++++
java/examples/pom.xml | 2 +-
pom.xml | 171 +++++++++++++++++++++++++++-
9 files changed, 418 insertions(+), 8 deletions(-)
diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml
new file mode 100644
index 00000000..29076c3c
--- /dev/null
+++ b/.github/workflows/code-coverage.yml
@@ -0,0 +1,35 @@
+name: Code Coverage
+on:
+ push:
+ branches:
+ - develop
+ paths-ignore:
+ - 'docs/**'
+ pull_request:
+ branches:
+ - develop
+ paths-ignore:
+ - 'docs/**'
+
+jobs:
+ codecov:
+ runs-on: ubuntu-latest
+ if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'apache/tsfile' || github.event_name == 'push'
+
+ steps:
+ - uses: actions/checkout@v4
+ - name: Cache Maven packages
+ uses: actions/cache@v4
+ with:
+ path: ~/.m2
+ key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+ restore-keys: ${{ runner.os }}-m2-
+ - name: Test
+ run: |
+ ./mvnw -B -P with-java,with-cpp,with-python,with-code-coverage clean verify
+ ./mvnw -B -P with-code-coverage post-integration-test
+ - name: Upload coverage reports to codecov
+ uses: codecov/codecov-action@v4
+ with:
+ token: ${{ secrets.CODECOV_TOKEN }}
+ files: ./code-coverage/target/jacoco-merged-reports/jacoco.xml
diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml
index 2192e465..93e95a80 100644
--- a/.github/workflows/unit-test.yml
+++ b/.github/workflows/unit-test.yml
@@ -38,7 +38,7 @@ jobs:
fail-fast: false
max-parallel: 20
matrix:
- java: [ 8, 11, 17, 21 ]
+ java: [ 8, 17, 21 ]
os: [ ubuntu-latest, macos-latest, windows-latest ]
runs-on: ${{ matrix.os }}
@@ -78,7 +78,7 @@ jobs:
}
# Run the actual maven build including all unit- and integration-tests.
- - name: Build and test with Maven (All others)
+ - name: Build and test with Maven
shell: bash
run: |
./mvnw${{ steps.platform_suffix.outputs.platform_suffix }} -P with-java,with-cpp,with-python clean verify
diff --git a/README-zh.md b/README-zh.md
index 8596329f..baf5bb9d 100644
--- a/README-zh.md
+++ b/README-zh.md
@@ -29,6 +29,7 @@ ___________ ___________.__.__
|____|/____ >\___ / |__|____/\___ > version 1.0.0
\/ \/ \/
</pre>
+[](https://codecov.io/github/apache/tsfile)
[](http://search.maven.org/#search|gav|1|g:"org.apache.tsfile")
## 简介
@@ -126,4 +127,4 @@ TsFile、CSV 和 Parquet 三种文件格式的比较
[C++](./cpp/README-zh.md)
-[Python](./python/README-zh.md)
\ No newline at end of file
+[Python](./python/README-zh.md)
diff --git a/README.md b/README.md
index 5b5636af..a5983790 100644
--- a/README.md
+++ b/README.md
@@ -29,6 +29,7 @@ ___________ ___________.__.__
|____|/____ >\___ / |__|____/\___ > version 1.0.0
\/ \/ \/
</pre>
+[](https://codecov.io/github/apache/tsfile)
[](http://search.maven.org/#search|gav|1|g:"org.apache.tsfile")
## Introduction
diff --git a/code-coverage/copy-code-coverage-sources.sh b/code-coverage/copy-code-coverage-sources.sh
new file mode 100755
index 00000000..19c24fd1
--- /dev/null
+++ b/code-coverage/copy-code-coverage-sources.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+for file in ../java/*/target/*.exec
+do
+ cp $file ./target/
+done
+
+for file in ../java/*/target/classes
+do
+ echo "copy from" ${file}
+ cp -r $file ./target/
+done
+
diff --git a/code-coverage/pom.xml b/code-coverage/pom.xml
new file mode 100644
index 00000000..b75bee1e
--- /dev/null
+++ b/code-coverage/pom.xml
@@ -0,0 +1,132 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.tsfile</groupId>
+ <artifactId>tsfile-parent</artifactId>
+ <version>1.0.1-SNAPSHOT</version>
+ </parent>
+ <artifactId>tsfile-code-coverage</artifactId>
+ <packaging>pom</packaging>
+ <name>TsFile: Code-Coverage</name>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>make-cmake-executable</id>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <phase>process-resources</phase>
+ <configuration>
+ <basedir>${project.basedir}</basedir>
+ <executable>chmod</executable>
+ <arguments>
+ <argument>+x</argument>
+ <argument>copy-code-coverage-sources.sh</argument>
+ </arguments>
+ </configuration>
+ </execution>
+ <execution>
+ <id>collect-code-coverage-results</id>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <phase>post-integration-test</phase>
+ <configuration>
+ <executable>${project.basedir}/copy-code-coverage-sources.sh</executable>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.jacoco</groupId>
+ <artifactId>jacoco-maven-plugin</artifactId>
+ <configuration>
+ <rules>
+ <rule implementation="org.jacoco.maven.RuleConfiguration">
+ <element>BUNDLE</element>
+ <limits>
+ <!-- Cover methodes >=30%. (the plugin does not support
+ ignore getter and setter and toString etc..) -->
+ <limit implementation="org.jacoco.report.check.Limit">
+ <counter>METHOD</counter>
+ <value>COVEREDRATIO</value>
+ <minimum>0.00</minimum>
+ </limit>
+ <!-- if-else, swtich etc.. >=70% -->
+ <limit implementation="org.jacoco.report.check.Limit">
+ <counter>BRANCH</counter>
+ <value>COVEREDRATIO</value>
+ <minimum>0.00</minimum>
+ </limit>
+ <!-- class files >=95% -->
+ <limit implementation="org.jacoco.report.check.Limit">
+ <counter>CLASS</counter>
+ <value>COVEREDRATIO</value>
+ <minimum>0.00</minimum>
+ </limit>
+ </limits>
+ </rule>
+ </rules>
+ </configuration>
+ <executions>
+ <!-- see https://natritmeyer.com/howto/reporting-aggregated-unit-and-integration-test-coverage-with-jacoco/-->
+ <execution>
+ <id>merge-unit-and-integration-final</id>
+ <goals>
+ <goal>merge</goal>
+ </goals>
+ <phase>post-integration-test</phase>
+ <configuration>
+ <fileSets>
+ <fileSet>
+ <directory>${project.build.directory}/</directory>
+ <includes>
+ <include>*.exec</include>
+ </includes>
+ </fileSet>
+ </fileSets>
+ <destFile>${project.build.directory}/merged.exec</destFile>
+ </configuration>
+ </execution>
+ <execution>
+ <id>create-merged-report-final</id>
+ <goals>
+ <goal>report</goal>
+ <goal>check</goal>
+ </goals>
+ <phase>post-integration-test</phase>
+ <configuration>
+ <dataFile>${project.build.directory}/merged.exec</dataFile>
+ <outputDirectory>${project.build.directory}/jacoco-merged-reports</outputDirectory>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/codecov.yml b/codecov.yml
new file mode 100644
index 00000000..aefe55fa
--- /dev/null
+++ b/codecov.yml
@@ -0,0 +1,47 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+coverage:
+ precision: 2
+ round: down
+ range: "50...100"
+
+ status:
+ project: yes
+ patch: no
+ changes: no
+
+parsers:
+ jacoco:
+ partials_as_hits: true #false by default
+
+ignore:
+ - "**/pom.xml"
+ - "**/*.md"
+ - "**/*.sh"
+ - "**/*.cmd"
+ - "**/*.bat"
+ - "**/*.yml"
+ - ".mvn"
+ - "mvnw"
+ - "NOTICE"
+ - "NOTICE-binary"
+ - "License"
+ - "LICENSE-binary"
+ - "DISCLAIMER"
diff --git a/java/examples/pom.xml b/java/examples/pom.xml
index 1f65e7bc..837be06a 100644
--- a/java/examples/pom.xml
+++ b/java/examples/pom.xml
@@ -28,7 +28,7 @@
</parent>
<artifactId>examples</artifactId>
<packaging>pom</packaging>
- <name>TSFile: Java: Examples</name>
+ <name>TsFile: Java: Examples</name>
<dependencies>
<dependency>
<groupId>ch.qos.logback</groupId>
diff --git a/pom.xml b/pom.xml
index 4ce3285b..197c0d7d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -608,7 +608,19 @@
[JEP 396: Strongly Encapsulate JDK Internals by Default],
[JEP 403: Strongly Encapsulate JDK Internals]
-->
- <argLine>--add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED --add [...]
+ <argLine><![CDATA[
+ --add-opens=java.base/java.lang=ALL-UNNAMED
+ --add-opens=java.base/java.util=ALL-UNNAMED
+ --add-opens=java.base/java.nio=ALL-UNNAMED
+ --add-opens=java.base/java.io=ALL-UNNAMED
+ --add-opens=java.base/java.net=ALL-UNNAMED
+ --add-opens=java.base/java.util.concurrent=ALL-UNNAMED
+ --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
+ --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
+ --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
+ --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
+ --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
+ ]]></argLine>
</properties>
</profile>
<!-- Current version of spotless cannot support JDK11 below -->
@@ -662,7 +674,7 @@
<python.compiler.argument/>
</properties>
</profile>
- <!-- Profile for linux aarch64 (mainly newer Mac or Raspberry PI Processors) (Self-Enabling) -->
+ <!-- Profile for linux aarch64 (mainly ARM64 Processors) (Self-Enabling) -->
<profile>
<id>.os-linux-aarch64</id>
<activation>
@@ -699,7 +711,7 @@
<python.compiler.argument/>
</properties>
</profile>
- <!-- Profile for mac aarch64 (mainly AMD Processors) (Self-Enabling) -->
+ <!-- Profile for mac aarch64 (mainly new Mac Processors) (Self-Enabling) -->
<profile>
<id>.os-mac-aarch64</id>
<activation>
@@ -757,7 +769,7 @@
<python.compiler.argument>--compiler=mingw32</python.compiler.argument>
</properties>
</profile>
- <!-- profile for windows aarch64 (mainly newer Mac or Raspberry PI Processors) (Self-Enabling) -->
+ <!-- profile for windows aarch64 (mainly VM on newer Mac) (Self-Enabling) -->
<profile>
<id>.os-windows-aarch64</id>
<activation>
@@ -875,6 +887,157 @@
</plugins>
</build>
</profile>
+ <profile>
+ <id>with-code-coverage</id>
+ <modules>
+ <module>code-coverage</module>
+ </modules>
+ <build>
+ <plugins>
+ <!--
+ Jacoco is a code coverage analysis plugin when tests run.
+ (not a static code analysis tool)
+ -->
+ <plugin>
+ <groupId>org.jacoco</groupId>
+ <artifactId>jacoco-maven-plugin</artifactId>
+ <version>0.8.12</version>
+ <configuration>
+ <rules>
+ <rule implementation="org.jacoco.maven.RuleConfiguration">
+ <element>BUNDLE</element>
+ <limits>
+ <!-- Cover methodes >=30%. (the plugin does not support
+ ignore getter and setter and toString etc..) -->
+ <limit implementation="org.jacoco.report.check.Limit">
+ <counter>METHOD</counter>
+ <value>COVEREDRATIO</value>
+ <minimum>0.00</minimum>
+ </limit>
+ <!-- if-else, swtich etc.. >=70% -->
+ <limit implementation="org.jacoco.report.check.Limit">
+ <counter>BRANCH</counter>
+ <value>COVEREDRATIO</value>
+ <minimum>0.00</minimum>
+ </limit>
+ <!-- class files >=95% -->
+ <limit implementation="org.jacoco.report.check.Limit">
+ <counter>CLASS</counter>
+ <value>COVEREDRATIO</value>
+ <minimum>0.00</minimum>
+ </limit>
+ </limits>
+ </rule>
+ </rules>
+ </configuration>
+ <executions>
+ <!-- For UT-->
+ <execution>
+ <id>prepare-ut</id>
+ <goals>
+ <goal>prepare-agent</goal>
+ </goals>
+ <configuration>
+ <destFile>${project.build.directory}/${project.build.finalName}-jacoco-unit-tests.exec</destFile>
+ <propertyName>surefire.jacoco.args</propertyName>
+ </configuration>
+ </execution>
+ <!-- attached to Maven test phase -->
+ <execution>
+ <id>ut-report</id>
+ <goals>
+ <goal>report</goal>
+ <goal>check</goal>
+ </goals>
+ <phase>test</phase>
+ <configuration>
+ <dataFile>${project.build.directory}/${project.build.finalName}-jacoco-unit-tests.exec</dataFile>
+ <outputDirectory>${project.build.directory}/jacoco-unit-reports</outputDirectory>
+ </configuration>
+ </execution>
+ <!-- For IT-->
+ <execution>
+ <id>before-integration-test-execution</id>
+ <goals>
+ <goal>prepare-agent</goal>
+ </goals>
+ <phase>pre-integration-test</phase>
+ <configuration>
+ <destFile>${project.build.directory}/${project.build.finalName}-jacoco-integration-tests.exec</destFile>
+ <propertyName>failsafe.jacoco.args</propertyName>
+ </configuration>
+ </execution>
+ <execution>
+ <id>after-integration-test-execution</id>
+ <goals>
+ <goal>report</goal>
+ <goal>check</goal>
+ </goals>
+ <phase>integration-test</phase>
+ <configuration>
+ <dataFile>${project.build.directory}/${project.build.finalName}-jacoco-integration-tests.exec</dataFile>
+ <outputDirectory>${project.build.directory}/jacoco-integration-reports</outputDirectory>
+ </configuration>
+ </execution>
+ <execution>
+ <id>merge-unit-and-integration</id>
+ <goals>
+ <goal>merge</goal>
+ </goals>
+ <phase>post-integration-test</phase>
+ <configuration>
+ <fileSets>
+ <fileSet>
+ <directory>${project.build.directory}/</directory>
+ <includes>
+ <include>*.exec</include>
+ </includes>
+ </fileSet>
+ </fileSets>
+ <destFile>${project.build.directory}/${project.build.finalName}-merged.exec</destFile>
+ </configuration>
+ </execution>
+ <execution>
+ <id>create-merged-report</id>
+ <goals>
+ <goal>report</goal>
+ <goal>check</goal>
+ </goals>
+ <phase>post-integration-test</phase>
+ <configuration>
+ <dataFile>${project.build.directory}/${project.build.finalName}-merged.exec</dataFile>
+ <outputDirectory>${project.build.directory}/jacoco-merged-reports</outputDirectory>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <!-- overwrite argLine-->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <argLine>${argLine} @{surefire.jacoco.args} -Xmx1024m</argLine>
+ </configuration>
+ </plugin>
+ <!-- for IT-->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <configuration>
+ <argLine>${argLine} @{failsafe.jacoco.args} -Xmx1024m</argLine>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>integration-test</goal>
+ <goal>verify</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
<profile>
<id>enforce</id>
<properties>
(tsfile) 11/13: Optimize code coverage structure (#195)
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch iotdb
in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit 3f241274f5a7f1b48b14a0d6560d6c23ca1e8d7d
Author: Haonan <hh...@outlook.com>
AuthorDate: Fri Aug 2 17:13:48 2024 +0800
Optimize code coverage structure (#195)
---
.github/workflows/code-coverage.yml | 2 +-
code-coverage/copy-code-coverage-sources.sh | 31 -------
code-coverage/pom.xml | 132 ----------------------------
pom.xml | 3 -
4 files changed, 1 insertion(+), 167 deletions(-)
diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml
index 3240ee0d..281ebe4c 100644
--- a/.github/workflows/code-coverage.yml
+++ b/.github/workflows/code-coverage.yml
@@ -37,4 +37,4 @@ jobs:
uses: codecov/codecov-action@v4
with:
token: ${{ secrets.CODECOV_TOKEN }}
- files: code-coverage/target/jacoco-merged-reports/jacoco.xml,cpp/target/build/test/coverage_report/index.html
+ files: java/tsfile/target/jacoco-merged-reports/jacoco.xml,cpp/target/build/test/coverage_report/index.html
diff --git a/code-coverage/copy-code-coverage-sources.sh b/code-coverage/copy-code-coverage-sources.sh
deleted file mode 100755
index 19c24fd1..00000000
--- a/code-coverage/copy-code-coverage-sources.sh
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/usr/bin/env bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-for file in ../java/*/target/*.exec
-do
- cp $file ./target/
-done
-
-for file in ../java/*/target/classes
-do
- echo "copy from" ${file}
- cp -r $file ./target/
-done
-
diff --git a/code-coverage/pom.xml b/code-coverage/pom.xml
deleted file mode 100644
index b75bee1e..00000000
--- a/code-coverage/pom.xml
+++ /dev/null
@@ -1,132 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.tsfile</groupId>
- <artifactId>tsfile-parent</artifactId>
- <version>1.0.1-SNAPSHOT</version>
- </parent>
- <artifactId>tsfile-code-coverage</artifactId>
- <packaging>pom</packaging>
- <name>TsFile: Code-Coverage</name>
- <build>
- <plugins>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>exec-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>make-cmake-executable</id>
- <goals>
- <goal>exec</goal>
- </goals>
- <phase>process-resources</phase>
- <configuration>
- <basedir>${project.basedir}</basedir>
- <executable>chmod</executable>
- <arguments>
- <argument>+x</argument>
- <argument>copy-code-coverage-sources.sh</argument>
- </arguments>
- </configuration>
- </execution>
- <execution>
- <id>collect-code-coverage-results</id>
- <goals>
- <goal>exec</goal>
- </goals>
- <phase>post-integration-test</phase>
- <configuration>
- <executable>${project.basedir}/copy-code-coverage-sources.sh</executable>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.jacoco</groupId>
- <artifactId>jacoco-maven-plugin</artifactId>
- <configuration>
- <rules>
- <rule implementation="org.jacoco.maven.RuleConfiguration">
- <element>BUNDLE</element>
- <limits>
- <!-- Cover methodes >=30%. (the plugin does not support
- ignore getter and setter and toString etc..) -->
- <limit implementation="org.jacoco.report.check.Limit">
- <counter>METHOD</counter>
- <value>COVEREDRATIO</value>
- <minimum>0.00</minimum>
- </limit>
- <!-- if-else, swtich etc.. >=70% -->
- <limit implementation="org.jacoco.report.check.Limit">
- <counter>BRANCH</counter>
- <value>COVEREDRATIO</value>
- <minimum>0.00</minimum>
- </limit>
- <!-- class files >=95% -->
- <limit implementation="org.jacoco.report.check.Limit">
- <counter>CLASS</counter>
- <value>COVEREDRATIO</value>
- <minimum>0.00</minimum>
- </limit>
- </limits>
- </rule>
- </rules>
- </configuration>
- <executions>
- <!-- see https://natritmeyer.com/howto/reporting-aggregated-unit-and-integration-test-coverage-with-jacoco/-->
- <execution>
- <id>merge-unit-and-integration-final</id>
- <goals>
- <goal>merge</goal>
- </goals>
- <phase>post-integration-test</phase>
- <configuration>
- <fileSets>
- <fileSet>
- <directory>${project.build.directory}/</directory>
- <includes>
- <include>*.exec</include>
- </includes>
- </fileSet>
- </fileSets>
- <destFile>${project.build.directory}/merged.exec</destFile>
- </configuration>
- </execution>
- <execution>
- <id>create-merged-report-final</id>
- <goals>
- <goal>report</goal>
- <goal>check</goal>
- </goals>
- <phase>post-integration-test</phase>
- <configuration>
- <dataFile>${project.build.directory}/merged.exec</dataFile>
- <outputDirectory>${project.build.directory}/jacoco-merged-reports</outputDirectory>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
diff --git a/pom.xml b/pom.xml
index 86c5651a..62de7afc 100644
--- a/pom.xml
+++ b/pom.xml
@@ -889,9 +889,6 @@
</profile>
<profile>
<id>with-code-coverage</id>
- <modules>
- <module>code-coverage</module>
- </modules>
<build>
<plugins>
<!--
(tsfile) 03/13: Bump org.tukaani:xz from 1.9 to 1.10 (#190)
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch iotdb
in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit 9be97f3833ea24bb2a2c4c0c9cbf0f5fb4516d27
Author: dependabot[bot] <49...@users.noreply.github.com>
AuthorDate: Thu Aug 1 15:01:41 2024 +0800
Bump org.tukaani:xz from 1.9 to 1.10 (#190)
Bumps [org.tukaani:xz](https://github.com/tukaani-project/xz-java) from 1.9 to 1.10.
- [Release notes](https://github.com/tukaani-project/xz-java/releases)
- [Changelog](https://github.com/tukaani-project/xz-java/blob/master/NEWS.md)
- [Commits](https://github.com/tukaani-project/xz-java/compare/v1.9...v1.10)
---
updated-dependencies:
- dependency-name: org.tukaani:xz
dependency-type: direct:production
update-type: version-update:semver-minor
...
Signed-off-by: dependabot[bot] <su...@github.com>
Co-authored-by: dependabot[bot] <49...@users.noreply.github.com>
---
java/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/java/pom.xml b/java/pom.xml
index a257c8c7..99b0565a 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -65,7 +65,7 @@
<dependency>
<groupId>org.tukaani</groupId>
<artifactId>xz</artifactId>
- <version>1.9</version>
+ <version>1.10</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
(tsfile) 05/13: Bump org.apache.maven.plugins:maven-javadoc-plugin from 3.7.0 to 3.8.0 (#186)
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch iotdb
in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit 684b9bea62effe2f3c34158493e11aa9e1242efd
Author: dependabot[bot] <49...@users.noreply.github.com>
AuthorDate: Thu Aug 1 15:02:24 2024 +0800
Bump org.apache.maven.plugins:maven-javadoc-plugin from 3.7.0 to 3.8.0 (#186)
Bumps [org.apache.maven.plugins:maven-javadoc-plugin](https://github.com/apache/maven-javadoc-plugin) from 3.7.0 to 3.8.0.
- [Release notes](https://github.com/apache/maven-javadoc-plugin/releases)
- [Commits](https://github.com/apache/maven-javadoc-plugin/compare/maven-javadoc-plugin-3.7.0...maven-javadoc-plugin-3.8.0)
---
updated-dependencies:
- dependency-name: org.apache.maven.plugins:maven-javadoc-plugin
dependency-type: direct:production
update-type: version-update:semver-minor
...
Signed-off-by: dependabot[bot] <su...@github.com>
Co-authored-by: dependabot[bot] <49...@users.noreply.github.com>
---
pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pom.xml b/pom.xml
index 197c0d7d..f5e97804 100644
--- a/pom.xml
+++ b/pom.xml
@@ -46,7 +46,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
- <version>3.7.0</version>
+ <version>3.8.0</version>
<configuration>
<!--
This will suppress the generation of a hidden timestamp at the top of each generated html page
(tsfile) 08/13: Fix c++ code coverage didn't show cc files (#193)
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch iotdb
in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit 15ed6590f7289245af2d5b431e67066fe27b398a
Author: Haonan <hh...@outlook.com>
AuthorDate: Thu Aug 1 23:20:10 2024 +0800
Fix c++ code coverage didn't show cc files (#193)
---
cpp/src/CMakeLists.txt | 3 +++
1 file changed, 3 insertions(+)
diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt
index ec07d19a..686520a0 100644
--- a/cpp/src/CMakeLists.txt
+++ b/cpp/src/CMakeLists.txt
@@ -18,6 +18,9 @@ under the License.
]]
message("Running in src diectory")
include(${PROJECT_SOURCE_DIR}/cmake/build_function.cmake)
+if (${COV_ENABLED})
+ add_compile_options(-fprofile-arcs -ftest-coverage)
+endif ()
add_subdirectory(common)
add_subdirectory(compress)
add_subdirectory(cwrapper)
(tsfile) 09/13: Bump org.codehaus.mojo:exec-maven-plugin from 1.6.0 to 3.3.0 (#189)
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch iotdb
in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit dde6b382de5fd14871d05b5d88f479bf80f5344d
Author: dependabot[bot] <49...@users.noreply.github.com>
AuthorDate: Thu Aug 1 23:30:05 2024 +0800
Bump org.codehaus.mojo:exec-maven-plugin from 1.6.0 to 3.3.0 (#189)
* Bump org.codehaus.mojo:exec-maven-plugin from 1.6.0 to 3.3.0
Bumps [org.codehaus.mojo:exec-maven-plugin](https://github.com/mojohaus/exec-maven-plugin) from 1.6.0 to 3.3.0.
- [Release notes](https://github.com/mojohaus/exec-maven-plugin/releases)
- [Commits](https://github.com/mojohaus/exec-maven-plugin/compare/exec-maven-plugin-1.6.0...3.3.0)
---
updated-dependencies:
- dependency-name: org.codehaus.mojo:exec-maven-plugin
dependency-type: direct:production
update-type: version-update:semver-major
...
Signed-off-by: dependabot[bot] <su...@github.com>
* fix error
---------
Signed-off-by: dependabot[bot] <su...@github.com>
Co-authored-by: dependabot[bot] <49...@users.noreply.github.com>
Co-authored-by: HTHou <hh...@outlook.com>
---
pom.xml | 18 +++++++++---------
python/pom.xml | 2 +-
2 files changed, 10 insertions(+), 10 deletions(-)
diff --git a/pom.xml b/pom.xml
index f5e97804..86c5651a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -197,7 +197,7 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
- <version>1.6.0</version>
+ <version>3.3.0</version>
</plugin>
<plugin>
<groupId>org.apache.felix</groupId>
@@ -652,7 +652,7 @@
<cmake.generator>Unix Makefiles</cmake.generator>
<python.venv.bin>venv/bin/</python.venv.bin>
<python.exe.bin>python3</python.exe.bin>
- <python.compiler.argument/>
+ <cython.compiler>unix</cython.compiler>
</properties>
</profile>
<!-- Profile for linux amd64 (mainly AMD Processors) (Self-Enabling) -->
@@ -671,7 +671,7 @@
<cmake.generator>Unix Makefiles</cmake.generator>
<python.venv.bin>venv/bin/</python.venv.bin>
<python.exe.bin>python3</python.exe.bin>
- <python.compiler.argument/>
+ <cython.compiler>unix</cython.compiler>
</properties>
</profile>
<!-- Profile for linux aarch64 (mainly ARM64 Processors) (Self-Enabling) -->
@@ -690,7 +690,7 @@
<cmake.generator>Unix Makefiles</cmake.generator>
<python.venv.bin>venv/bin/</python.venv.bin>
<python.exe.bin>python3</python.exe.bin>
- <python.compiler.argument/>
+ <cython.compiler>unix</cython.compiler>
</properties>
</profile>
<!-- Profile for mac x86_64 (mainly Intel Processors) (Self-Enabling) -->
@@ -708,7 +708,7 @@
<cmake.generator>Unix Makefiles</cmake.generator>
<python.venv.bin>venv/bin/</python.venv.bin>
<python.exe.bin>python3</python.exe.bin>
- <python.compiler.argument/>
+ <cython.compiler>unix</cython.compiler>
</properties>
</profile>
<!-- Profile for mac aarch64 (mainly new Mac Processors) (Self-Enabling) -->
@@ -726,7 +726,7 @@
<cmake.generator>Unix Makefiles</cmake.generator>
<python.venv.bin>venv/bin/</python.venv.bin>
<python.exe.bin>python3</python.exe.bin>
- <python.compiler.argument/>
+ <cython.compiler>unix</cython.compiler>
</properties>
</profile>
<!-- profile for windows x86_64 (mainly Intel Processors) (Self-Enabling) -->
@@ -746,7 +746,7 @@
<!--cmake.generator>Visual Studio 17 2022</cmake.generator-->
<python.venv.bin>venv/Scripts/</python.venv.bin>
<python.exe.bin>python</python.exe.bin>
- <python.compiler.argument>--compiler=mingw32</python.compiler.argument>
+ <cython.compiler>mingw32</cython.compiler>
</properties>
</profile>
<!-- profile for windows amd64 (mainly AMD Processors) (Self-Enabling) -->
@@ -766,7 +766,7 @@
<!--cmake.generator>Visual Studio 17 2022</cmake.generator-->
<python.venv.bin>venv/Scripts/</python.venv.bin>
<python.exe.bin>python</python.exe.bin>
- <python.compiler.argument>--compiler=mingw32</python.compiler.argument>
+ <cython.compiler>mingw32</cython.compiler>
</properties>
</profile>
<!-- profile for windows aarch64 (mainly VM on newer Mac) (Self-Enabling) -->
@@ -786,7 +786,7 @@
<!--cmake.generator>Visual Studio 17 2022</cmake.generator-->
<python.venv.bin>venv/Scripts/</python.venv.bin>
<python.exe.bin>python</python.exe.bin>
- <python.compiler.argument>--compiler=mingw32</python.compiler.argument>
+ <cython.compiler>mingw32</cython.compiler>
</properties>
</profile>
<!-- Little helper profile that will disable running the cmake tests when the maven tests are being skipped -->
diff --git a/python/pom.xml b/python/pom.xml
index cd5f2df4..09a61c71 100644
--- a/python/pom.xml
+++ b/python/pom.xml
@@ -103,7 +103,7 @@
<argument>setup.py</argument>
<argument>build_ext</argument>
<argument>--inplace</argument>
- <argument>${python.compiler.argument}</argument>
+ <argument>--compiler=${cython.compiler}</argument>
</arguments>
</configuration>
</execution>
(tsfile) 10/13: Use pytest to run python test (#194)
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch iotdb
in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit 77db2076caa9a5bdf7c61959f66dc3a701a69979
Author: Haonan <hh...@outlook.com>
AuthorDate: Fri Aug 2 00:57:04 2024 +0800
Use pytest to run python test (#194)
---
.github/workflows/code-coverage.yml | 2 +-
python/pom.xml | 18 +++++++++++++++++-
python/test.py | 2 +-
3 files changed, 19 insertions(+), 3 deletions(-)
diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml
index f1a4f116..3240ee0d 100644
--- a/.github/workflows/code-coverage.yml
+++ b/.github/workflows/code-coverage.yml
@@ -27,7 +27,7 @@ jobs:
- name: Generate code coverage reports
run: |
sudo apt-get install lcov
- ./mvnw -B -P with-java,with-cpp,with-code-coverage clean verify
+ ./mvnw -B -P with-java,with-cpp,with-python,with-code-coverage clean verify
lcov --capture --directory cpp/target/build/test --output-file cpp/target/build/test/coverage.info
lcov --remove cpp/target/build/test/coverage.info '*/tsfile/cpp/test/*' --output-file cpp/target/build/test/coverage_filtered.info
genhtml cpp/target/build/test/coverage_filtered.info --output-directory cpp/target/build/test/coverage_report
diff --git a/python/pom.xml b/python/pom.xml
index 09a61c71..7791e832 100644
--- a/python/pom.xml
+++ b/python/pom.xml
@@ -107,6 +107,22 @@
</arguments>
</configuration>
</execution>
+ <execution>
+ <id>python-install-test-requirements</id>
+ <phase>process-test-resources</phase>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <configuration>
+ <executable>${python.venv.bin}${python.exe.bin}</executable>
+ <arguments>
+ <argument>-m</argument>
+ <argument>pip</argument>
+ <argument>install</argument>
+ <argument>pytest</argument>
+ </arguments>
+ </configuration>
+ </execution>
<execution>
<id>run-python-tests</id>
<phase>test</phase>
@@ -114,7 +130,7 @@
<goal>exec</goal>
</goals>
<configuration>
- <executable>${python.venv.bin}${python.exe.bin}</executable>
+ <executable>${python.venv.bin}pytest</executable>
<arguments>
<argument>${project.basedir}/test.py</argument>
</arguments>
diff --git a/python/test.py b/python/test.py
index e4838742..29fb6c24 100644
--- a/python/test.py
+++ b/python/test.py
@@ -29,7 +29,7 @@ import tsfile as ts
from tsfile.tsfile import EmptyFileError
TABLE_NAME = "test_table"
-DATA_PATH = os.path.join(os.path.dirname(__file__), "data")
+DATA_PATH = os.path.join(os.path.dirname(__file__), "target")
# test writing data
(tsfile) 06/13: Generate code coverage report for c++ code (#191)
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch iotdb
in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit b45db1002d62572fbee6ad4f1e9f5f1fcb2e63f7
Author: Haonan <hh...@outlook.com>
AuthorDate: Thu Aug 1 21:17:23 2024 +0800
Generate code coverage report for c++ code (#191)
---
.github/workflows/code-coverage.yml | 13 +++++++++----
codecov.yml | 7 +++++++
cpp/CMakeLists.txt | 10 +++++++---
cpp/pom.xml | 10 ++++++++++
cpp/src/CMakeLists.txt | 11 ++++++++---
cpp/test/CMakeLists.txt | 7 ++++++-
6 files changed, 47 insertions(+), 11 deletions(-)
diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml
index 29076c3c..f1a4f116 100644
--- a/.github/workflows/code-coverage.yml
+++ b/.github/workflows/code-coverage.yml
@@ -24,12 +24,17 @@ jobs:
path: ~/.m2
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
restore-keys: ${{ runner.os }}-m2-
- - name: Test
+ - name: Generate code coverage reports
run: |
- ./mvnw -B -P with-java,with-cpp,with-python,with-code-coverage clean verify
- ./mvnw -B -P with-code-coverage post-integration-test
+ sudo apt-get install lcov
+ ./mvnw -B -P with-java,with-cpp,with-code-coverage clean verify
+ lcov --capture --directory cpp/target/build/test --output-file cpp/target/build/test/coverage.info
+ lcov --remove cpp/target/build/test/coverage.info '*/tsfile/cpp/test/*' --output-file cpp/target/build/test/coverage_filtered.info
+ genhtml cpp/target/build/test/coverage_filtered.info --output-directory cpp/target/build/test/coverage_report
+ rm -rf cpp/target/build/test/CMakeFiles/TsFile_Test.dir
+ rm -rf cpp/target/build/test/coverage.info
- name: Upload coverage reports to codecov
uses: codecov/codecov-action@v4
with:
token: ${{ secrets.CODECOV_TOKEN }}
- files: ./code-coverage/target/jacoco-merged-reports/jacoco.xml
+ files: code-coverage/target/jacoco-merged-reports/jacoco.xml,cpp/target/build/test/coverage_report/index.html
diff --git a/codecov.yml b/codecov.yml
index aefe55fa..7c21286f 100644
--- a/codecov.yml
+++ b/codecov.yml
@@ -30,6 +30,12 @@ coverage:
parsers:
jacoco:
partials_as_hits: true #false by default
+ gcov:
+ branch_detection:
+ conditional: yes
+ loop: yes
+ method: no
+ macro: no
ignore:
- "**/pom.xml"
@@ -38,6 +44,7 @@ ignore:
- "**/*.cmd"
- "**/*.bat"
- "**/*.yml"
+ - "*_test.cc"
- ".mvn"
- "mvnw"
- "NOTICE"
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index afca5202..185b3905 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -33,7 +33,7 @@ if(DEFINED ENV{CXX})
message("cmake using: CXX=${CMAKE_CXX_COMPILER}")
endif()
-if(DEFINED ENV{CC})
+if(DEFINED ENV{CC})
set(CMAKE_C_COMPILER $ENV{CC})
message("cmake using: CC=${CMAKE_C_COMPILER}")
endif()
@@ -44,6 +44,11 @@ if (${DEBUG_SE})
message("add_definitions -DDEBUG_SE=1")
endif()
+if (${COV_ENABLED})
+ add_definitions(-DCOV_ENABLED=1)
+ message("add_definitions -DCOV_ENABLED=1")
+endif()
+
message("build type: ${BUILD_TYPE}")
if (CMAKE_BUILD_TYPE STREQUAL "Debug")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g")
@@ -74,8 +79,7 @@ configure_file(
"${PROJECT_BINARY_DIR}/cmake_config.h"
)
-set(TESTS_ENABLED OFF)
add_subdirectory(test)
if(TESTS_ENABLED)
add_dependencies(TsFile_Test tsfile)
-endif()
\ No newline at end of file
+endif()
diff --git a/cpp/pom.xml b/cpp/pom.xml
index faad3610..2527014d 100644
--- a/cpp/pom.xml
+++ b/cpp/pom.xml
@@ -32,6 +32,7 @@
<!-- Tell Sonar where to find the sources -->
<sonar.sources>common,examples,tsfile</sonar.sources>
<sonar.cfamily.build-wrapper-output>${project.build.directory}/build-wrapper-output</sonar.cfamily.build-wrapper-output>
+ <cmake.addition.option>-DCOV_ENABLED=OFF</cmake.addition.option>
</properties>
<build>
<sourceDirectory>${project.basedir}</sourceDirectory>
@@ -69,7 +70,10 @@
<configuration>
<options>
<option>-DBUILD_PHASE=test-compile</option>
+ <option>${cmake.addition.option}</option>
</options>
+ <sourcePath/>
+ <targetPath/>
</configuration>
</execution>
<!-- Compile the test code -->
@@ -119,6 +123,12 @@
</plugins>
</build>
<profiles>
+ <profile>
+ <id>with-code-coverage</id>
+ <properties>
+ <cmake.addition.option>-DCOV_ENABLED=ON</cmake.addition.option>
+ </properties>
+ </profile>
<!-- When running on jenkins, download the sonar build-wrapper, so we can do a code analysis -->
<profile>
<id>jenkins-build</id>
diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt
index 9f323c77..ec07d19a 100644
--- a/cpp/src/CMakeLists.txt
+++ b/cpp/src/CMakeLists.txt
@@ -18,7 +18,6 @@ under the License.
]]
message("Running in src diectory")
include(${PROJECT_SOURCE_DIR}/cmake/build_function.cmake)
-
add_subdirectory(common)
add_subdirectory(compress)
add_subdirectory(cwrapper)
@@ -29,7 +28,13 @@ add_subdirectory(utils)
add_subdirectory(writer)
add_library(tsfile SHARED)
-target_link_libraries(tsfile common_obj compress_obj cwrapper_obj file_obj read_obj write_obj )
+if (${COV_ENABLED})
+ message("Enable code cov...")
+ target_link_libraries(tsfile common_obj compress_obj cwrapper_obj file_obj read_obj write_obj -lgcov)
+else()
+ message("Disable code cov...")
+ target_link_libraries(tsfile common_obj compress_obj cwrapper_obj file_obj read_obj write_obj)
+endif()
set(LIBTSFILE_PROJECT_VERSION 1.0)
set(LIBTSFILE_SO_VERSION 1.0)
@@ -53,4 +58,4 @@ install(TARGETS tsfile LIBRARY DESTINATION ${LIBTSFILE_SDK_DIR})
# if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
# add_custom_command(TARGET tsfile POST_BUILD COMMAND ${CMAKE_INSTALL_NAME_TOOL} -change `otool -L ${LIBRARY_OUTPUT_PATH}/libtsfile.dylib | grep liblz4 | sed 's/dylib.*/dylib/g'` ${my_lz4_lib} ${LIBRARY_OUTPUT_PATH}/libtsfile.dylib)
# add_custom_command(TARGET tsfile POST_BUILD COMMAND ${CMAKE_INSTALL_NAME_TOOL} -change `otool -L ${LIBRARY_OUTPUT_PATH}/libtsfile.dylib | grep libz | sed 's/dylib.*/dylib/g'` ${my_z_lib} ${LIBRARY_OUTPUT_PATH}/libtsfile.dylib)
-# endif()
\ No newline at end of file
+# endif()
diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt
index 91fa71c5..df6d3015 100644
--- a/cpp/test/CMakeLists.txt
+++ b/cpp/test/CMakeLists.txt
@@ -74,6 +74,11 @@ file(GLOB_RECURSE TEST_SRCS
"compress/*_test.cc"
"writer/*_test.cc"
)
+if (${COV_ENABLED})
+ message("Enable code cov...")
+ add_compile_options(-fprofile-arcs -ftest-coverage)
+endif ()
+
add_executable(TsFile_Test ${TEST_SRCS})
target_link_libraries(
TsFile_Test
@@ -84,4 +89,4 @@ target_link_libraries(
set_target_properties(TsFile_Test PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${LIBTSFILE_SDK_DIR})
include(GoogleTest)
-gtest_discover_tests(TsFile_Test)
\ No newline at end of file
+gtest_discover_tests(TsFile_Test)