You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ro...@apache.org on 2023/07/28 19:24:31 UTC
[iotdb] branch rel/1.1 updated: [IOTDB-6078] Load: Allow different chunks have different compressors and encoders & Load time chunk using compressor recorded in chunk header (#10649) (#10735)
This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch rel/1.1
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/1.1 by this push:
new 270d49a6e87 [IOTDB-6078] Load: Allow different chunks have different compressors and encoders & Load time chunk using compressor recorded in chunk header (#10649) (#10735)
270d49a6e87 is described below
commit 270d49a6e876f12487eba099138a2c5ebecd8d4a
Author: Itami Sho <42...@users.noreply.github.com>
AuthorDate: Sat Jul 29 03:24:25 2023 +0800
[IOTDB-6078] Load: Allow different chunks have different compressors and encoders & Load time chunk using compressor recorded in chunk header (#10649) (#10735)
(cherry picked from commit aafb5831feb2f9aef9852d7bb4fe0c9d39acbeb8)
---
.../org/apache/iotdb/db/engine/load/AlignedChunkData.java | 9 ++++++++-
.../org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 12 ++++++------
.../iotdb/tsfile/write/chunk/AlignedChunkWriterImpl.java | 2 +-
3 files changed, 15 insertions(+), 8 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/load/AlignedChunkData.java b/server/src/main/java/org/apache/iotdb/db/engine/load/AlignedChunkData.java
index cdd40664655..b9ca6db8e5f 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/load/AlignedChunkData.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/load/AlignedChunkData.java
@@ -277,8 +277,15 @@ public class AlignedChunkData implements ChunkData {
private void buildChunkWriter(InputStream stream) throws IOException, PageException {
List<IMeasurementSchema> measurementSchemaList = new ArrayList<>();
+ IMeasurementSchema timeSchema = null;
for (ChunkHeader chunkHeader : chunkHeaderList) {
if (TSDataType.VECTOR.equals(chunkHeader.getDataType())) {
+ timeSchema =
+ new MeasurementSchema(
+ chunkHeader.getMeasurementID(),
+ chunkHeader.getDataType(),
+ chunkHeader.getEncodingType(),
+ chunkHeader.getCompressionType());
continue;
}
measurementSchemaList.add(
@@ -288,7 +295,7 @@ public class AlignedChunkData implements ChunkData {
chunkHeader.getEncodingType(),
chunkHeader.getCompressionType()));
}
- chunkWriter = new AlignedChunkWriterImpl(measurementSchemaList);
+ chunkWriter = new AlignedChunkWriterImpl(timeSchema, measurementSchemaList);
timeBatch = new ArrayList<>();
int chunkHeaderSize = chunkHeaderList.size();
for (int i = 0; i < chunkHeaderSize; i++) {
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java
index b24db1ecb44..b699978b539 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java
@@ -2314,19 +2314,19 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
originSchema.getType().name());
}
if (!tsFileSchema.getEncodingType().equals(originSchema.getEncodingType())) {
- throw new VerifyMetadataException(
+ logger.warn(
+ "Encoding type not match, measurement: {}, TsFile: {}, TsFile encoding: {}, origin encoding: {}",
measurementPath,
- "Encoding",
- tsFileSchema.getEncodingType().name(),
entry.getValue().get(tsFileSchema).getPath(),
+ tsFileSchema.getEncodingType().name(),
originSchema.getEncodingType().name());
}
if (!tsFileSchema.getCompressor().equals(originSchema.getCompressor())) {
- throw new VerifyMetadataException(
+ logger.warn(
+ "Compression type not match, measurement: {}, TsFile: {}, TsFile compression: {}, origin compression: {}",
measurementPath,
- "Compress type",
- tsFileSchema.getCompressor().name(),
entry.getValue().get(tsFileSchema).getPath(),
+ tsFileSchema.getCompressor().name(),
originSchema.getCompressor().name());
}
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/AlignedChunkWriterImpl.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/AlignedChunkWriterImpl.java
index fbf124eba5a..f4b5794dc20 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/AlignedChunkWriterImpl.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/AlignedChunkWriterImpl.java
@@ -110,7 +110,7 @@ public class AlignedChunkWriterImpl implements IChunkWriter {
}
/**
- * This is used to write 0-level file. The compression of the time column is 'SNAPPY' in the
+ * This is used to write 0-level file. The compression of the time column is 'LZ4' in the
* configuration by default. The encoding of the time column is 'TS_2DIFF' in the configuration by
* default.
*