You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2023/06/05 09:06:14 UTC
[iotdb] branch ty/benchant updated (659b08be5ac -> 5d6c41e0ce5)
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a change to branch ty/benchant
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 659b08be5ac remove cglib dependency
add 03d0feb6647 Added ITs for logical views & Fix duplicate rows in show views. (#10024)
add 6a6d805c4d3 remove max pool size metric (#10032)
add 2021d5aacc2 Get canonical path when using IOTDB_DATA_HOME and IOTDB_HOME (#10035)
add 322c9c68dbe [IOTDB-5965] Pipe: failed to transfer insert node when using schema template (#10041)
add ed42e91e047 Fix SchemaTree.hasView check during query analysis (#10040)
add 48afbc54be7 Add schema_ engine_mode config to iotdb-commons.properties and User-Guide (#10048)
add 5d78f1a915f [IOTDB-5968] Pipe: pipe task does not work properly after cluster reboot (#10046)
add 562048d4800 [IOTDB-5966] Pipe: Revision of Tablet serialize and deserialize (#10045)
add 387c2102600 [IOTDB-5967] Pipe: fix convertToTablet bug and introduce PipeEmptyTabletInsertionEvent (#10044)
new 91330db04b3 Merge remote-tracking branch 'origin/master' into ty/benchant
new 5d6c41e0ce5 update cache in flush and compacting
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../iotdb/confignode/manager/node/NodeManager.java | 9 +-
.../confignode/persistence/pipe/PipeTaskInfo.java | 9 +-
.../runtime/PipeHandleLeaderChangeProcedure.java | 8 +-
.../runtime/PipeHandleMetaChangeProcedure.java | 24 +-
.../impl/pipe/runtime/PipeMetaSyncProcedure.java | 8 +-
.../pipe/task/AbstractOperatePipeProcedureV2.java | 8 +
docs/UserGuide/Reference/Common-Config-Manual.md | 9 +
.../zh/UserGuide/Reference/Common-Config-Manual.md | 9 +
.../iotdb/db/it/view/IoTDBCreateAndShowViewIT.java | 294 ++++++++++++++
.../iotdb/pipe/api/collector/RowCollector.java | 5 +-
.../resources/conf/iotdb-common.properties | 5 +
.../commons/concurrent/ThreadPoolMetrics.java | 16 -
.../apache/iotdb/commons/conf/CommonConfig.java | 31 +-
.../consensus/index/impl/SimpleProgressIndex.java | 2 +-
.../iotdb/commons/pipe/task/meta/PipeTaskMeta.java | 5 +-
.../org/apache/iotdb/commons/utils/FileUtils.java | 12 +
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 21 +-
.../iotdb/db/engine/cache/BloomFilterCache.java | 4 +
.../db/engine/cache/TimeSeriesMetadataCache.java | 8 +
.../impl/ReadChunkCompactionPerformer.java | 3 +-
.../writer/AbstractCrossCompactionWriter.java | 3 +-
.../writer/AbstractInnerCompactionWriter.java | 4 +-
.../db/engine/storagegroup/TsFileProcessor.java | 3 +-
.../db/metadata/schemaregion/SchemaEngine.java | 2 -
.../metadata/schemaregion/SchemaRegionLoader.java | 77 +---
.../mpp/common/schematree/ClusterSchemaTree.java | 4 +-
.../db/pipe/agent/runtime/HeartbeatScheduler.java | 23 --
.../db/pipe/agent/runtime/PipeAgentLauncher.java | 12 +-
.../iotdb/db/pipe/agent/task/PipeTaskAgent.java | 51 ++-
.../impl/iotdb/v1/IoTDBThriftConnectorV1.java | 38 +-
.../v1/request/PipeTransferInsertNodeReq.java | 4 +-
.../event/impl/PipeEmptyTabletInsertionEvent.java | 28 +-
...ava => PipeInsertNodeTabletInsertionEvent.java} | 31 +-
...nt.java => PipeTabletTabletInsertionEvent.java} | 6 +-
.../realtime/PipeRealtimeCollectEventFactory.java | 6 +-
.../core/event/realtime/TsFileEpochManager.java | 12 +-
.../db/pipe/core/event/view/access/PipeRow.java | 22 +-
.../event/view/collector/PipeRowCollector.java | 10 +-
.../TabletInsertionDataContainer.java | 106 ++----
.../TsFileInsertionDataContainer.java | 4 +-
.../UpdateCacheRestorableTsFileIOWriter.java | 58 +++
.../db/query/reader/UpdateCacheTsFileIOWriter.java | 51 +++
.../db/sync/common/ClusterSyncInfoFetcher.java | 13 +-
.../file/AbstractTsFileRecoverPerformer.java | 3 +-
.../META-INF/native-image/reflect-config.json | 364 +-----------------
.../META-INF/native-image/resource-config.json | 3 -
.../utils/CompactionFileGeneratorUtils.java | 10 +-
.../common/schematree/ClusterSchemaTreeTest.java | 21 +
.../PipeInsertNodeTabletInsertionEventTest.java | 178 +++++++++
.../tsfile/file/metadata/TimeseriesMetadata.java | 4 +-
.../apache/iotdb/tsfile/write/record/Tablet.java | 422 +++++++++++++--------
.../iotdb/tsfile/write/writer/TsFileIOWriter.java | 11 +
.../write/writer/tsmiterator/TSMIterator.java | 18 +-
.../iotdb/tsfile/write/record/TabletTest.java | 38 ++
54 files changed, 1263 insertions(+), 867 deletions(-)
create mode 100644 integration-test/src/test/java/org/apache/iotdb/db/it/view/IoTDBCreateAndShowViewIT.java
delete mode 100644 server/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/HeartbeatScheduler.java
copy iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/collector/RowCollector.java => server/src/main/java/org/apache/iotdb/db/pipe/core/event/impl/PipeEmptyTabletInsertionEvent.java (65%)
rename server/src/main/java/org/apache/iotdb/db/pipe/core/event/impl/{PipeInsertNodeInsertionEvent.java => PipeInsertNodeTabletInsertionEvent.java} (83%)
rename server/src/main/java/org/apache/iotdb/db/pipe/core/event/impl/{PipeTabletInsertionEvent.java => PipeTabletTabletInsertionEvent.java} (92%)
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/reader/UpdateCacheRestorableTsFileIOWriter.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/reader/UpdateCacheTsFileIOWriter.java
create mode 100644 server/src/test/java/org/apache/iotdb/db/pipe/core/event/PipeInsertNodeTabletInsertionEventTest.java
[iotdb] 01/02: Merge remote-tracking branch 'origin/master' into ty/benchant
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch ty/benchant
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 91330db04b3992db860ae0fc759f921a2bb253b3
Merge: 659b08be5ac 387c2102600
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Mon Jun 5 14:02:50 2023 +0800
Merge remote-tracking branch 'origin/master' into ty/benchant
.../iotdb/confignode/manager/node/NodeManager.java | 9 +-
.../confignode/persistence/pipe/PipeTaskInfo.java | 9 +-
.../runtime/PipeHandleLeaderChangeProcedure.java | 8 +-
.../runtime/PipeHandleMetaChangeProcedure.java | 24 +-
.../impl/pipe/runtime/PipeMetaSyncProcedure.java | 8 +-
.../pipe/task/AbstractOperatePipeProcedureV2.java | 8 +
docs/UserGuide/Reference/Common-Config-Manual.md | 9 +
.../zh/UserGuide/Reference/Common-Config-Manual.md | 9 +
.../iotdb/db/it/view/IoTDBCreateAndShowViewIT.java | 294 ++++++++++++++
.../iotdb/pipe/api/collector/RowCollector.java | 5 +-
.../resources/conf/iotdb-common.properties | 5 +
.../commons/concurrent/ThreadPoolMetrics.java | 16 -
.../apache/iotdb/commons/conf/CommonConfig.java | 31 +-
.../consensus/index/impl/SimpleProgressIndex.java | 2 +-
.../iotdb/commons/pipe/task/meta/PipeTaskMeta.java | 5 +-
.../org/apache/iotdb/commons/utils/FileUtils.java | 12 +
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 21 +-
.../mpp/common/schematree/ClusterSchemaTree.java | 4 +-
.../db/pipe/agent/runtime/PipeAgentLauncher.java | 12 +-
.../iotdb/db/pipe/agent/task/PipeTaskAgent.java | 51 ++-
.../impl/iotdb/v1/IoTDBThriftConnectorV1.java | 38 +-
.../v1/request/PipeTransferInsertNodeReq.java | 4 +-
.../event/impl/PipeEmptyTabletInsertionEvent.java} | 22 +-
...ava => PipeInsertNodeTabletInsertionEvent.java} | 31 +-
...nt.java => PipeTabletTabletInsertionEvent.java} | 6 +-
.../realtime/PipeRealtimeCollectEventFactory.java | 6 +-
.../core/event/realtime/TsFileEpochManager.java | 12 +-
.../db/pipe/core/event/view/access/PipeRow.java | 22 +-
.../event/view/collector/PipeRowCollector.java | 10 +-
.../TabletInsertionDataContainer.java | 106 ++----
.../TsFileInsertionDataContainer.java | 4 +-
.../db/sync/common/ClusterSyncInfoFetcher.java | 13 +-
.../common/schematree/ClusterSchemaTreeTest.java | 21 +
.../PipeInsertNodeTabletInsertionEventTest.java | 178 +++++++++
.../apache/iotdb/tsfile/write/record/Tablet.java | 422 +++++++++++++--------
.../iotdb/tsfile/write/record/TabletTest.java | 38 ++
36 files changed, 1091 insertions(+), 384 deletions(-)
[iotdb] 02/02: update cache in flush and compacting
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch ty/benchant
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 5d6c41e0ce5be5bdd3268a211ed265a24404e62f
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Mon Jun 5 17:05:59 2023 +0800
update cache in flush and compacting
---
.../iotdb/db/engine/cache/BloomFilterCache.java | 4 +
.../db/engine/cache/TimeSeriesMetadataCache.java | 8 +
.../impl/ReadChunkCompactionPerformer.java | 3 +-
.../writer/AbstractCrossCompactionWriter.java | 3 +-
.../writer/AbstractInnerCompactionWriter.java | 4 +-
.../db/engine/storagegroup/TsFileProcessor.java | 3 +-
.../db/metadata/schemaregion/SchemaEngine.java | 2 -
.../metadata/schemaregion/SchemaRegionLoader.java | 77 +----
.../UpdateCacheRestorableTsFileIOWriter.java | 58 ++++
.../db/query/reader/UpdateCacheTsFileIOWriter.java | 51 +++
.../file/AbstractTsFileRecoverPerformer.java | 3 +-
.../META-INF/native-image/reflect-config.json | 364 +--------------------
.../META-INF/native-image/resource-config.json | 3 -
.../utils/CompactionFileGeneratorUtils.java | 10 +-
.../tsfile/file/metadata/TimeseriesMetadata.java | 4 +-
.../iotdb/tsfile/write/writer/TsFileIOWriter.java | 11 +
.../write/writer/tsmiterator/TSMIterator.java | 18 +-
17 files changed, 179 insertions(+), 447 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/BloomFilterCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/BloomFilterCache.java
index b1d5b0b5b40..6a2f6b196d8 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/BloomFilterCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/BloomFilterCache.java
@@ -125,6 +125,10 @@ public class BloomFilterCache {
return entryAverageSize.get();
}
+ public void updateCache(String filePath, BloomFilter bloomFilter) {
+ lruCache.put(new BloomFilterCacheKey(filePath), bloomFilter);
+ }
+
/** clear LRUCache. */
public void clear() {
lruCache.invalidateAll();
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
index 823b47be2c8..0f4e69567d1 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
@@ -238,6 +238,14 @@ public class TimeSeriesMetadataCache {
return BloomFilterCache.getInstance().calculateBloomFilterHitRatio();
}
+ public void updateCache(String filePath, String deviceId, TimeseriesMetadata timeseriesMetadata) {
+ if (timeseriesMetadata.getStatistics().getCount() != 0) {
+ lruCache.put(
+ new TimeSeriesMetadataCacheKey(filePath, deviceId, timeseriesMetadata.getMeasurementId()),
+ timeseriesMetadata);
+ }
+ }
+
/** clear LRUCache. */
public void clear() {
lruCache.invalidateAll();
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java
index da0c9c91dd8..8c0dcb463f1 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java
@@ -29,6 +29,7 @@ import org.apache.iotdb.db.engine.compaction.execute.utils.executor.readchunk.Al
import org.apache.iotdb.db.engine.compaction.execute.utils.executor.readchunk.SingleSeriesCompactionExecutor;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.exception.StorageEngineException;
+import org.apache.iotdb.db.query.reader.UpdateCacheTsFileIOWriter;
import org.apache.iotdb.db.rescon.SystemInfo;
import org.apache.iotdb.tsfile.file.metadata.AlignedChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
@@ -72,7 +73,7 @@ public class ReadChunkCompactionPerformer implements ISeqCompactionPerformer {
* IoTDBDescriptor.getInstance().getConfig().getChunkMetadataSizeProportion());
try (MultiTsFileDeviceIterator deviceIterator = new MultiTsFileDeviceIterator(seqFiles);
TsFileIOWriter writer =
- new TsFileIOWriter(targetResource.getTsFile(), true, sizeForFileWriter)) {
+ new UpdateCacheTsFileIOWriter(targetResource.getTsFile(), true, sizeForFileWriter)) {
while (deviceIterator.hasNextDevice()) {
Pair<String, Boolean> deviceInfo = deviceIterator.nextDevice();
String device = deviceInfo.left;
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/writer/AbstractCrossCompactionWriter.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/writer/AbstractCrossCompactionWriter.java
index 44409808daf..3e19bf05b39 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/writer/AbstractCrossCompactionWriter.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/writer/AbstractCrossCompactionWriter.java
@@ -22,6 +22,7 @@ import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.compaction.execute.utils.CompactionUtils;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.db.query.reader.UpdateCacheTsFileIOWriter;
import org.apache.iotdb.db.rescon.SystemInfo;
import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.iotdb.tsfile.read.TimeValuePair;
@@ -77,7 +78,7 @@ public abstract class AbstractCrossCompactionWriter extends AbstractCompactionWr
boolean enableMemoryControl = IoTDBDescriptor.getInstance().getConfig().isEnableMemControl();
for (int i = 0; i < targetResources.size(); i++) {
this.targetFileWriters.add(
- new TsFileIOWriter(
+ new UpdateCacheTsFileIOWriter(
targetResources.get(i).getTsFile(), enableMemoryControl, memorySizeForEachWriter));
isEmptyFile[i] = true;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/writer/AbstractInnerCompactionWriter.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/writer/AbstractInnerCompactionWriter.java
index 6404730d0e4..326cf8ad8e1 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/writer/AbstractInnerCompactionWriter.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/writer/AbstractInnerCompactionWriter.java
@@ -21,6 +21,7 @@ package org.apache.iotdb.db.engine.compaction.execute.utils.writer;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.compaction.execute.utils.CompactionUtils;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.db.query.reader.UpdateCacheTsFileIOWriter;
import org.apache.iotdb.db.rescon.SystemInfo;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.read.TimeValuePair;
@@ -50,7 +51,8 @@ public abstract class AbstractInnerCompactionWriter extends AbstractCompactionWr
* IoTDBDescriptor.getInstance().getConfig().getChunkMetadataSizeProportion());
boolean enableMemoryControl = IoTDBDescriptor.getInstance().getConfig().isEnableMemControl();
this.fileWriter =
- new TsFileIOWriter(targetFileResource.getTsFile(), enableMemoryControl, sizeForFileWriter);
+ new UpdateCacheTsFileIOWriter(
+ targetFileResource.getTsFile(), enableMemoryControl, sizeForFileWriter);
this.targetResource = targetFileResource;
isEmptyFile = true;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
index cbc609e3605..e0649cda0ad 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
@@ -55,6 +55,7 @@ import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertRowNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertTabletNode;
import org.apache.iotdb.db.pipe.core.collector.realtime.listener.PipeInsertionDataNodeListener;
import org.apache.iotdb.db.query.context.QueryContext;
+import org.apache.iotdb.db.query.reader.UpdateCacheRestorableTsFileIOWriter;
import org.apache.iotdb.db.rescon.MemTableManager;
import org.apache.iotdb.db.rescon.PrimitiveArrayManager;
import org.apache.iotdb.db.rescon.SystemInfo;
@@ -194,7 +195,7 @@ public class TsFileProcessor {
this.sequence = sequence;
this.tsFileResource = new TsFileResource(tsfile, this);
this.dataRegionInfo = dataRegionInfo;
- this.writer = new RestorableTsFileIOWriter(tsfile);
+ this.writer = new UpdateCacheRestorableTsFileIOWriter(tsfile);
this.updateLatestFlushTimeCallback = updateLatestFlushTimeCallback;
this.walNode =
WALManager.getInstance()
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngine.java b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngine.java
index e8f3fdfbb21..8fbe4ea4dca 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngine.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngine.java
@@ -93,8 +93,6 @@ public class SchemaEngine {
public void init() {
logger.info("used schema engine mode: {}.", config.getSchemaEngineMode());
- schemaRegionLoader.init(config.getSchemaEngineMode());
-
initSchemaEngineStatistics();
SchemaResourceManager.initSchemaResource(schemaEngineStatistics);
// CachedSchemaEngineMetric depend on CacheMemoryManager, so it should be initialized after
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionLoader.java b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionLoader.java
index 0eab047f8d3..c7a29aad07c 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionLoader.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionLoader.java
@@ -20,18 +20,12 @@
package org.apache.iotdb.db.metadata.schemaregion;
import org.apache.iotdb.commons.exception.MetadataException;
-import org.apache.iotdb.db.metadata.MetadataConstant;
-import org.reflections.Reflections;
-import org.reflections.util.ConfigurationBuilder;
-import org.reflections.util.FilterBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
import java.util.Map;
-import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
class SchemaRegionLoader {
@@ -43,82 +37,15 @@ class SchemaRegionLoader {
private String currentMode;
- private Constructor<ISchemaRegion> currentConstructor;
-
@SuppressWarnings("unchecked")
- SchemaRegionLoader() {
- Reflections reflections =
- new Reflections(
- new ConfigurationBuilder()
- .forPackages(PACKAGE_NAME)
- .filterInputsBy(new FilterBuilder().includePackage(PACKAGE_NAME)));
-
- Set<Class<?>> annotatedSchemaRegionSet = reflections.getTypesAnnotatedWith(SchemaRegion.class);
-
- for (Class<?> annotatedSchemaRegion : annotatedSchemaRegionSet) {
- boolean isSchemaRegion = false;
- for (Class<?> interfaces : annotatedSchemaRegion.getInterfaces()) {
- if (interfaces == ISchemaRegion.class) {
- isSchemaRegion = true;
- break;
- }
- }
- if (!isSchemaRegion) {
- logger.warn(
- String.format(
- "Class %s is not a subclass of ISchemaRegion.", annotatedSchemaRegion.getName()));
- continue;
- }
- SchemaRegion annotationInfo = annotatedSchemaRegion.getAnnotation(SchemaRegion.class);
- constructorMap.compute(
- annotationInfo.mode(),
- (k, v) -> {
- if (v == null) {
- try {
- return (Constructor<ISchemaRegion>)
- annotatedSchemaRegion.getConstructor(ISchemaRegionParams.class);
- } catch (NoSuchMethodException e) {
- logger.error(e.getMessage(), e);
- return null;
- }
- }
- logger.warn(
- "Duplicated SchemaRegion implementation, {} and {}, with same mode name [{}]",
- v.getClass().getName(),
- annotatedSchemaRegion.getName(),
- k);
- return v;
- });
- }
- }
-
- void init(String schemaEngineMode) {
- Constructor<ISchemaRegion> constructor = constructorMap.get(schemaEngineMode);
- if (constructor == null) {
- logger.warn(
- "There's no SchemaRegion implementation with target mode {}. Use default mode {}",
- schemaEngineMode,
- MetadataConstant.DEFAULT_SCHEMA_ENGINE_MODE);
- currentMode = MetadataConstant.DEFAULT_SCHEMA_ENGINE_MODE;
- currentConstructor = constructorMap.get(currentMode);
- } else {
- currentMode = schemaEngineMode;
- currentConstructor = constructor;
- }
- }
+ SchemaRegionLoader() {}
ISchemaRegion createSchemaRegion(ISchemaRegionParams schemaRegionParams)
throws MetadataException {
- try {
- return currentConstructor.newInstance(schemaRegionParams);
- } catch (InstantiationException | IllegalAccessException | InvocationTargetException e) {
- logger.warn(e.getMessage(), e);
- throw new MetadataException(e);
- }
+ return new SchemaRegionMemoryImpl(schemaRegionParams);
}
void clear() {
currentMode = null;
- currentConstructor = null;
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/UpdateCacheRestorableTsFileIOWriter.java b/server/src/main/java/org/apache/iotdb/db/query/reader/UpdateCacheRestorableTsFileIOWriter.java
new file mode 100644
index 00000000000..cefbb412fa9
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/UpdateCacheRestorableTsFileIOWriter.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.query.reader;
+
+import org.apache.iotdb.db.engine.cache.BloomFilterCache;
+import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache;
+import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
+import org.apache.iotdb.tsfile.utils.BloomFilter;
+import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter;
+
+import java.io.File;
+import java.io.IOException;
+
+public class UpdateCacheRestorableTsFileIOWriter extends RestorableTsFileIOWriter {
+
+ private static final BloomFilterCache BLOOM_FILTER_CACHE = BloomFilterCache.getInstance();
+
+ private static final TimeSeriesMetadataCache TIME_SERIES_METADATA_CACHE =
+ TimeSeriesMetadataCache.getInstance();
+
+ public UpdateCacheRestorableTsFileIOWriter(File file) throws IOException {
+ super(file);
+ }
+
+ public UpdateCacheRestorableTsFileIOWriter(File file, long maxMetadataSize) throws IOException {
+ super(file, maxMetadataSize);
+ }
+
+ public UpdateCacheRestorableTsFileIOWriter(File file, boolean truncate) throws IOException {
+ super(file, truncate);
+ }
+
+ @Override
+ protected void updateCache(String device, TimeseriesMetadata timeseriesMetadata) {
+ TIME_SERIES_METADATA_CACHE.updateCache(file.getPath(), device, timeseriesMetadata);
+ }
+
+ @Override
+ protected void updateCache(BloomFilter bloomFilter) {
+ BLOOM_FILTER_CACHE.updateCache(file.getPath(), bloomFilter);
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/UpdateCacheTsFileIOWriter.java b/server/src/main/java/org/apache/iotdb/db/query/reader/UpdateCacheTsFileIOWriter.java
new file mode 100644
index 00000000000..12ef2a7b79e
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/UpdateCacheTsFileIOWriter.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.query.reader;
+
+import org.apache.iotdb.db.engine.cache.BloomFilterCache;
+import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache;
+import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
+import org.apache.iotdb.tsfile.utils.BloomFilter;
+import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter;
+
+import java.io.File;
+import java.io.IOException;
+
+public class UpdateCacheTsFileIOWriter extends TsFileIOWriter {
+
+ public UpdateCacheTsFileIOWriter(File file, boolean enableMemoryControl, long maxMetadataSize)
+ throws IOException {
+ super(file, enableMemoryControl, maxMetadataSize);
+ }
+
+ private static final BloomFilterCache BLOOM_FILTER_CACHE = BloomFilterCache.getInstance();
+
+ private static final TimeSeriesMetadataCache TIME_SERIES_METADATA_CACHE =
+ TimeSeriesMetadataCache.getInstance();
+
+ @Override
+ protected void updateCache(String device, TimeseriesMetadata timeseriesMetadata) {
+ TIME_SERIES_METADATA_CACHE.updateCache(file.getPath(), device, timeseriesMetadata);
+ }
+
+ @Override
+ protected void updateCache(BloomFilter bloomFilter) {
+ BLOOM_FILTER_CACHE.updateCache(file.getPath(), bloomFilter);
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/wal/recover/file/AbstractTsFileRecoverPerformer.java b/server/src/main/java/org/apache/iotdb/db/wal/recover/file/AbstractTsFileRecoverPerformer.java
index aa08daf4151..e305c4def2b 100644
--- a/server/src/main/java/org/apache/iotdb/db/wal/recover/file/AbstractTsFileRecoverPerformer.java
+++ b/server/src/main/java/org/apache/iotdb/db/wal/recover/file/AbstractTsFileRecoverPerformer.java
@@ -20,6 +20,7 @@ package org.apache.iotdb.db.wal.recover.file;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.exception.DataRegionException;
+import org.apache.iotdb.db.query.reader.UpdateCacheRestorableTsFileIOWriter;
import org.apache.iotdb.db.utils.FileLoaderUtils;
import org.apache.iotdb.tsfile.exception.NotCompatibleTsFileException;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
@@ -77,7 +78,7 @@ public abstract class AbstractTsFileRecoverPerformer implements Closeable {
// try to remove corrupted part of the TsFile
try {
- writer = new RestorableTsFileIOWriter(tsFile);
+ writer = new UpdateCacheRestorableTsFileIOWriter(tsFile);
} catch (NotCompatibleTsFileException e) {
boolean result = tsFile.delete();
logger.warn(
diff --git a/server/src/main/resources/META-INF/native-image/reflect-config.json b/server/src/main/resources/META-INF/native-image/reflect-config.json
index 545175af084..989ba09286d 100644
--- a/server/src/main/resources/META-INF/native-image/reflect-config.json
+++ b/server/src/main/resources/META-INF/native-image/reflect-config.json
@@ -23,9 +23,6 @@
{
"name":"[Ljavax.management.openmbean.CompositeData;"
},
-{
- "name":"[Lnet.sf.cglib.proxy.Callback;"
-},
{
"name":"[Lorg.HdrHistogram.DoubleRecorder;"
},
@@ -229,28 +226,12 @@
"name":"com.sun.org.apache.xerces.internal.jaxp.SAXParserFactoryImpl",
"methods":[{"name":"<init>","parameterTypes":[] }]
},
-{
- "name":"java.io.DataInputStream"
-},
-{
- "name":"java.io.DataOutputStream"
-},
{
"name":"java.io.FileDescriptor"
},
{
"name":"java.io.FilePermission"
},
-{
- "name":"java.io.InputStream"
-},
-{
- "name":"java.io.Serializable"
-},
-{
- "name":"java.lang.AutoCloseable",
- "queryAllDeclaredMethods":true
-},
{
"name":"java.lang.Boolean",
"allDeclaredFields":true,
@@ -266,16 +247,6 @@
"allDeclaredFields":true,
"fields":[{"name":"TYPE"}]
},
-{
- "name":"java.lang.Class"
-},
-{
- "name":"java.lang.ClassLoader",
- "methods":[{"name":"defineClass","parameterTypes":["java.lang.String","byte[]","int","int","java.security.ProtectionDomain"] }]
-},
-{
- "name":"java.lang.Comparable"
-},
{
"name":"java.lang.Deprecated",
"queryAllPublicMethods":true
@@ -285,9 +256,6 @@
"allDeclaredFields":true,
"fields":[{"name":"TYPE"}]
},
-{
- "name":"java.lang.Enum"
-},
{
"name":"java.lang.Float",
"allDeclaredFields":true,
@@ -303,17 +271,13 @@
"allDeclaredFields":true,
"fields":[{"name":"TYPE"}]
},
-{
- "name":"java.lang.Module"
-},
{
"name":"java.lang.Number",
"allDeclaredFields":true
},
{
"name":"java.lang.Object",
- "allDeclaredFields":true,
- "queryAllDeclaredMethods":true
+ "allDeclaredFields":true
},
{
"name":"java.lang.ProcessHandle",
@@ -338,9 +302,6 @@
"name":"java.lang.String",
"allDeclaredFields":true
},
-{
- "name":"java.lang.StringBuilder"
-},
{
"name":"java.lang.Thread",
"fields":[{"name":"threadLocalRandomProbe"}]
@@ -353,12 +314,6 @@
"name":"java.lang.Void",
"fields":[{"name":"TYPE"}]
},
-{
- "name":"java.lang.annotation.Annotation"
-},
-{
- "name":"java.lang.invoke.CallSite"
-},
{
"name":"java.lang.management.BufferPoolMXBean",
"queryAllPublicMethods":true
@@ -475,19 +430,12 @@
{
"name":"java.security.AllPermission"
},
-{
- "name":"java.security.ProtectionDomain"
-},
{
"name":"java.security.SecureRandomParameters"
},
{
"name":"java.security.SecurityPermission"
},
-{
- "name":"java.util.AbstractMap",
- "allDeclaredFields":true
-},
{
"name":"java.util.AbstractMap$SimpleEntry",
"allDeclaredFields":true
@@ -499,17 +447,9 @@
{
"name":"java.util.Date"
},
-{
- "name":"java.util.Iterator"
-},
-{
- "name":"java.util.List",
- "methods":[{"name":"copyOf","parameterTypes":["java.util.Collection"] }]
-},
{
"name":"java.util.Optional",
- "allDeclaredFields":true,
- "methods":[{"name":"isEmpty","parameterTypes":[] }]
+ "allDeclaredFields":true
},
{
"name":"java.util.OptionalDouble",
@@ -527,22 +467,6 @@
"name":"java.util.PropertyPermission",
"methods":[{"name":"<init>","parameterTypes":["java.lang.String","java.lang.String"] }]
},
-{
- "name":"java.util.concurrent.ConcurrentHashMap",
- "allDeclaredFields":true
-},
-{
- "name":"java.util.concurrent.ConcurrentHashMap$CollectionView",
- "allDeclaredFields":true
-},
-{
- "name":"java.util.concurrent.ConcurrentHashMap$EntrySetView",
- "allDeclaredFields":true
-},
-{
- "name":"java.util.concurrent.ConcurrentHashMap$Node",
- "allDeclaredFields":true
-},
{
"name":"java.util.concurrent.atomic.LongAdder",
"queryAllPublicConstructors":true,
@@ -552,9 +476,6 @@
{"name":"sum","parameterTypes":[] }
]
},
-{
- "name":"java.util.function.Function"
-},
{
"name":"java.util.logging.LogManager",
"methods":[{"name":"getLoggingMXBean","parameterTypes":[] }]
@@ -563,9 +484,6 @@
"name":"java.util.logging.LoggingMXBean",
"queryAllPublicMethods":true
},
-{
- "name":"java.util.zip.DeflaterInputStream"
-},
{
"name":"javax.management.MBeanOperationInfo",
"queryAllPublicMethods":true,
@@ -618,62 +536,6 @@
"name":"jdk.management.jfr.SettingDescriptorInfo",
"queryAllPublicMethods":true
},
-{
- "name":"net.sf.cglib.core.KeyFactory"
-},
-{
- "name":"net.sf.cglib.core.MethodWrapper$MethodWrapperKey",
- "queryAllDeclaredMethods":true
-},
-{
- "name":"net.sf.cglib.core.MethodWrapper$MethodWrapperKey$$KeyFactoryByCGLIB$$d45e49f7",
- "methods":[{"name":"<init>","parameterTypes":[] }]
-},
-{
- "name":"net.sf.cglib.core.ReflectUtils"
-},
-{
- "name":"net.sf.cglib.core.Signature"
-},
-{
- "name":"net.sf.cglib.proxy.Dispatcher"
-},
-{
- "name":"net.sf.cglib.proxy.Enhancer"
-},
-{
- "name":"net.sf.cglib.proxy.Enhancer$EnhancerKey",
- "queryAllDeclaredMethods":true
-},
-{
- "name":"net.sf.cglib.proxy.Enhancer$EnhancerKey$$KeyFactoryByCGLIB$$7fb24d72",
- "methods":[{"name":"<init>","parameterTypes":[] }]
-},
-{
- "name":"net.sf.cglib.proxy.Factory",
- "queryAllDeclaredMethods":true
-},
-{
- "name":"net.sf.cglib.proxy.FixedValue"
-},
-{
- "name":"net.sf.cglib.proxy.InvocationHandler"
-},
-{
- "name":"net.sf.cglib.proxy.LazyLoader"
-},
-{
- "name":"net.sf.cglib.proxy.MethodInterceptor"
-},
-{
- "name":"net.sf.cglib.proxy.NoOp"
-},
-{
- "name":"net.sf.cglib.proxy.ProxyRefDispatcher"
-},
-{
- "name":"net.sf.cglib.reflect.FastClass"
-},
{
"name":"org.HdrHistogram.ConcurrentHistogram",
"methods":[{"name":"<init>","parameterTypes":["long","long","int"] }]
@@ -702,10 +564,6 @@
"name":"org.apache.iotdb.commons.auth.authorizer.LocalFileAuthorizer",
"methods":[{"name":"<init>","parameterTypes":[] }]
},
-{
- "name":"org.apache.iotdb.commons.client.ThriftClient",
- "queryAllDeclaredMethods":true
-},
{
"name":"org.apache.iotdb.commons.concurrent.threadpool.WrappedScheduledExecutorService",
"queryAllPublicConstructors":true
@@ -742,60 +600,6 @@
"name":"org.apache.iotdb.commons.partition.executor.hash.BKDRHashExecutor",
"methods":[{"name":"<init>","parameterTypes":["int"] }]
},
-{
- "name":"org.apache.iotdb.commons.path.PathPatternNode$Serializer"
-},
-{
- "name":"org.apache.iotdb.commons.schema.filter.SchemaFilterVisitor"
-},
-{
- "name":"org.apache.iotdb.commons.schema.node.IMNode"
-},
-{
- "name":"org.apache.iotdb.commons.schema.node.common.AbstractAboveDatabaseMNode"
-},
-{
- "name":"org.apache.iotdb.commons.schema.node.common.AbstractDatabaseDeviceMNode"
-},
-{
- "name":"org.apache.iotdb.commons.schema.node.common.AbstractDatabaseMNode"
-},
-{
- "name":"org.apache.iotdb.commons.schema.node.common.AbstractDeviceMNode"
-},
-{
- "name":"org.apache.iotdb.commons.schema.node.common.AbstractMeasurementMNode"
-},
-{
- "name":"org.apache.iotdb.commons.schema.node.info.IDatabaseDeviceInfo"
-},
-{
- "name":"org.apache.iotdb.commons.schema.node.info.IDatabaseInfo"
-},
-{
- "name":"org.apache.iotdb.commons.schema.node.info.IDeviceInfo"
-},
-{
- "name":"org.apache.iotdb.commons.schema.node.info.IMeasurementInfo"
-},
-{
- "name":"org.apache.iotdb.commons.schema.node.utils.IMNodeContainer"
-},
-{
- "name":"org.apache.iotdb.commons.schema.node.utils.IMNodeFactory"
-},
-{
- "name":"org.apache.iotdb.commons.schema.node.utils.IMNodeIterator"
-},
-{
- "name":"org.apache.iotdb.commons.schema.node.visitor.MNodeVisitor"
-},
-{
- "name":"org.apache.iotdb.commons.schema.tree.AbstractTreeVisitor"
-},
-{
- "name":"org.apache.iotdb.commons.schema.view.viewExpression.visitor.ViewExpressionVisitor"
-},
{
"name":"org.apache.iotdb.commons.security.encrypt.MessageDigestEncrypt",
"methods":[{"name":"<init>","parameterTypes":[] }]
@@ -808,10 +612,6 @@
"name":"org.apache.iotdb.commons.service.metric.MetricServiceMBean",
"queryAllPublicMethods":true
},
-{
- "name":"org.apache.iotdb.confignode.rpc.thrift.IConfigNodeRPCService$Iface",
- "queryAllDeclaredMethods":true
-},
{
"name":"org.apache.iotdb.consensus.iot.IoTConsensus",
"methods":[{"name":"<init>","parameterTypes":["org.apache.iotdb.consensus.config.ConsensusConfig","org.apache.iotdb.consensus.IStateMachine$Registry"] }]
@@ -828,29 +628,6 @@
"name":"org.apache.iotdb.consensus.ratis.RatisConsensus",
"methods":[{"name":"<init>","parameterTypes":["org.apache.iotdb.consensus.config.ConsensusConfig","org.apache.iotdb.consensus.IStateMachine$Registry"] }]
},
-{
- "name":"org.apache.iotdb.db.client.ConfigNodeClient",
- "queryAllDeclaredMethods":true,
- "queryAllDeclaredConstructors":true,
- "methods":[{"name":"<init>","parameterTypes":["java.util.List","org.apache.iotdb.commons.client.property.ThriftClientProperty","org.apache.iotdb.commons.client.ClientManager"] }]
-},
-{
- "name":"org.apache.iotdb.db.client.ConfigNodeClient$$EnhancerByCGLIB$$e2d55b5e",
- "queryAllDeclaredMethods":true,
- "queryAllDeclaredConstructors":true,
- "methods":[
- {"name":"<init>","parameterTypes":["java.util.List","org.apache.iotdb.commons.client.property.ThriftClientProperty","org.apache.iotdb.commons.client.ClientManager"] },
- {"name":"CGLIB$SET_THREAD_CALLBACKS","parameterTypes":["net.sf.cglib.proxy.Callback[]"] }
- ]
-},
-{
- "name":"org.apache.iotdb.db.client.ConfigNodeClient$$EnhancerByCGLIB$$e2d55b5e$$FastClassByCGLIB$$90aef4e7",
- "methods":[{"name":"<init>","parameterTypes":["java.lang.Class"] }]
-},
-{
- "name":"org.apache.iotdb.db.client.ConfigNodeClient$$FastClassByCGLIB$$c29c157a",
- "methods":[{"name":"<init>","parameterTypes":["java.lang.Class"] }]
-},
{
"name":"org.apache.iotdb.db.conf.IoTDBConfig",
"allDeclaredFields":true
@@ -866,6 +643,10 @@
"name":"org.apache.iotdb.db.engine.cache.CacheHitRatioMonitorMXBean",
"queryAllPublicMethods":true
},
+{
+ "name":"org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache$TimeSeriesMetadataCacheKey",
+ "allDeclaredFields":true
+},
{
"name":"org.apache.iotdb.db.engine.flush.FlushManager",
"queryAllPublicConstructors":true
@@ -874,122 +655,6 @@
"name":"org.apache.iotdb.db.engine.flush.FlushManagerMBean",
"queryAllPublicMethods":true
},
-{
- "name":"org.apache.iotdb.db.metadata.cache.dualkeycache.IDualKeyCache"
-},
-{
- "name":"org.apache.iotdb.db.metadata.cache.dualkeycache.IDualKeyCacheComputation"
-},
-{
- "name":"org.apache.iotdb.db.metadata.cache.dualkeycache.IDualKeyCacheStats"
-},
-{
- "name":"org.apache.iotdb.db.metadata.cache.dualkeycache.impl.ICacheEntry"
-},
-{
- "name":"org.apache.iotdb.db.metadata.cache.dualkeycache.impl.ICacheEntryGroup"
-},
-{
- "name":"org.apache.iotdb.db.metadata.cache.dualkeycache.impl.ICacheEntryManager"
-},
-{
- "name":"org.apache.iotdb.db.metadata.cache.dualkeycache.impl.ICacheSizeComputer"
-},
-{
- "name":"org.apache.iotdb.db.metadata.cache.lastCache.container.ILastCacheContainer"
-},
-{
- "name":"org.apache.iotdb.db.metadata.cache.lastCache.container.value.ILastCacheValue"
-},
-{
- "name":"org.apache.iotdb.db.metadata.idtable.IDTable"
-},
-{
- "name":"org.apache.iotdb.db.metadata.idtable.IDiskSchemaManager"
-},
-{
- "name":"org.apache.iotdb.db.metadata.idtable.entry.IDeviceID"
-},
-{
- "name":"org.apache.iotdb.db.metadata.logfile.IDeserializer"
-},
-{
- "name":"org.apache.iotdb.db.metadata.logfile.ISerializer"
-},
-{
- "name":"org.apache.iotdb.db.metadata.mnode.mem.info.BasicMNodeInfo"
-},
-{
- "name":"org.apache.iotdb.db.metadata.mtree.store.IMTreeStore"
-},
-{
- "name":"org.apache.iotdb.db.metadata.mtree.store.disk.cache.CacheEntry"
-},
-{
- "name":"org.apache.iotdb.db.metadata.mtree.store.disk.cache.ICacheManager"
-},
-{
- "name":"org.apache.iotdb.db.metadata.mtree.store.disk.memcontrol.IReleaseFlushStrategy"
-},
-{
- "name":"org.apache.iotdb.db.metadata.mtree.store.disk.schemafile.ISchemaFile"
-},
-{
- "name":"org.apache.iotdb.db.metadata.mtree.store.disk.schemafile.ISchemaPage"
-},
-{
- "name":"org.apache.iotdb.db.metadata.mtree.store.disk.schemafile.ISegment"
-},
-{
- "name":"org.apache.iotdb.db.metadata.mtree.store.disk.schemafile.pagemgr.IPageManager"
-},
-{
- "name":"org.apache.iotdb.db.metadata.mtree.traverser.counter.Counter"
-},
-{
- "name":"org.apache.iotdb.db.metadata.mtree.traverser.updater.Updater"
-},
-{
- "name":"org.apache.iotdb.db.metadata.plan.schemaregion.ISchemaRegionPlan"
-},
-{
- "name":"org.apache.iotdb.db.metadata.plan.schemaregion.SchemaRegionPlanVisitor"
-},
-{
- "name":"org.apache.iotdb.db.metadata.query.info.ISchemaInfo"
-},
-{
- "name":"org.apache.iotdb.db.metadata.rescon.ISchemaEngineStatistics"
-},
-{
- "name":"org.apache.iotdb.db.metadata.rescon.ISchemaRegionStatistics"
-},
-{
- "name":"org.apache.iotdb.db.metadata.schemaregion.ISchemaRegion"
-},
-{
- "name":"org.apache.iotdb.db.metadata.schemaregion.ISchemaRegionParams"
-},
-{
- "name":"org.apache.iotdb.db.metadata.schemaregion.SchemaRegionMemoryImpl",
- "methods":[{"name":"<init>","parameterTypes":["org.apache.iotdb.db.metadata.schemaregion.ISchemaRegionParams"] }]
-},
-{
- "name":"org.apache.iotdb.db.metadata.schemaregion.SchemaRegionSchemaFileImpl",
- "methods":[{"name":"<init>","parameterTypes":["org.apache.iotdb.db.metadata.schemaregion.ISchemaRegionParams"] }]
-},
-{
- "name":"org.apache.iotdb.db.metadata.template.ITemplateManager"
-},
-{
- "name":"org.apache.iotdb.db.metadata.template.alter.TemplateAlterInfo"
-},
-{
- "name":"org.apache.iotdb.db.metadata.utils.ResourceByPathUtils"
-},
-{
- "name":"org.apache.iotdb.db.mpp.common.schematree.IMeasurementSchemaInfo"
-},
{
"name":"org.apache.iotdb.db.mpp.execution.exchange.MPPDataExchangeService",
"queryAllPublicConstructors":true
@@ -1002,9 +667,6 @@
"name":"org.apache.iotdb.db.mpp.execution.operator.process.last.LastQueryMergeOperator$Location",
"allDeclaredFields":true
},
-{
- "name":"org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanVisitor"
-},
{
"name":"org.apache.iotdb.db.query.control.SessionManager",
"queryAllPublicConstructors":true
@@ -1046,7 +708,16 @@
"methods":[{"name":"loadAll","parameterTypes":["java.lang.Iterable"] }]
},
{
- "name":"org.apache.iotdb.metrics.metricsets.IMetricSet"
+ "name":"org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata",
+ "allDeclaredFields":true
+},
+{
+ "name":"org.apache.iotdb.tsfile.file.metadata.statistics.LongStatistics",
+ "allDeclaredFields":true
+},
+{
+ "name":"org.apache.iotdb.tsfile.file.metadata.statistics.Statistics",
+ "allDeclaredFields":true
},
{
"name":"org.apache.iotdb.tsfile.read.common.block.TsBlock",
@@ -1464,9 +1135,6 @@
{"name":"f4"}
]
},
-{
- "name":"org.slf4j.impl.StaticLoggerBinder"
-},
{
"name":"sun.management.ClassLoadingImpl",
"queryAllPublicConstructors":true
diff --git a/server/src/main/resources/META-INF/native-image/resource-config.json b/server/src/main/resources/META-INF/native-image/resource-config.json
index 2960797dd0a..c11048a2ab2 100644
--- a/server/src/main/resources/META-INF/native-image/resource-config.json
+++ b/server/src/main/resources/META-INF/native-image/resource-config.json
@@ -19,9 +19,6 @@
{
"pattern":"\\Qlogback-test.xml\\E"
},
- {
- "pattern":"\\Qorg/apache/iotdb/db/metadata\\E"
- },
{
"pattern":"\\Qorg/slf4j/impl/StaticLoggerBinder.class\\E"
},
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/CompactionFileGeneratorUtils.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/CompactionFileGeneratorUtils.java
index a5cd18a2974..fdfc65e750c 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/CompactionFileGeneratorUtils.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/CompactionFileGeneratorUtils.java
@@ -28,6 +28,7 @@ import org.apache.iotdb.db.engine.modification.ModificationFile;
import org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator;
import org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator.TsFileName;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.db.query.reader.UpdateCacheRestorableTsFileIOWriter;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
@@ -181,7 +182,8 @@ public class CompactionFileGeneratorUtils {
if (!newTsFileResource.getTsFile().getParentFile().exists()) {
newTsFileResource.getTsFile().getParentFile().mkdirs();
}
- RestorableTsFileIOWriter writer = new RestorableTsFileIOWriter(newTsFileResource.getTsFile());
+ RestorableTsFileIOWriter writer =
+ new UpdateCacheRestorableTsFileIOWriter(newTsFileResource.getTsFile());
Map<String, List<String>> deviceMeasurementMap = new HashMap<>();
for (String fullPath : fullPaths) {
PartialPath partialPath = new PartialPath(fullPath);
@@ -238,7 +240,8 @@ public class CompactionFileGeneratorUtils {
TSFileDescriptor.getInstance().getConfig().getMaxNumberOfPointsInPage();
TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(Integer.MAX_VALUE);
- RestorableTsFileIOWriter writer = new RestorableTsFileIOWriter(newTsFileResource.getTsFile());
+ RestorableTsFileIOWriter writer =
+ new UpdateCacheRestorableTsFileIOWriter(newTsFileResource.getTsFile());
Map<String, List<String>> deviceMeasurementMap = new HashMap<>();
for (String fullPath : fullPaths) {
PartialPath partialPath = new PartialPath(fullPath);
@@ -329,7 +332,8 @@ public class CompactionFileGeneratorUtils {
if (!newTsFileResource.getTsFile().getParentFile().exists()) {
newTsFileResource.getTsFile().getParentFile().mkdirs();
}
- RestorableTsFileIOWriter writer = new RestorableTsFileIOWriter(newTsFileResource.getTsFile());
+ RestorableTsFileIOWriter writer =
+ new UpdateCacheRestorableTsFileIOWriter(newTsFileResource.getTsFile());
Map<String, List<String>> deviceMeasurementMap = new HashMap<>();
for (String fullPath : fullPaths) {
PartialPath partialPath = new PartialPath(fullPath);
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
index 2dd9b211232..f59e2de6829 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
@@ -82,13 +82,15 @@ public class TimeseriesMetadata implements ITimeSeriesMetadata {
String measurementId,
TSDataType dataType,
Statistics<? extends Serializable> statistics,
- PublicBAOS chunkMetadataListBuffer) {
+ PublicBAOS chunkMetadataListBuffer,
+ ArrayList<IChunkMetadata> chunkMetadataList) {
this.timeSeriesMetadataType = timeSeriesMetadataType;
this.chunkMetaDataListDataSize = chunkMetaDataListDataSize;
this.measurementId = measurementId;
this.dataType = dataType;
this.statistics = statistics;
this.chunkMetadataListBuffer = chunkMetadataListBuffer;
+ this.chunkMetadataList = chunkMetadataList;
}
public TimeseriesMetadata(TimeseriesMetadata timeseriesMetadata) {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
index e62aeb40c7e..57786c2f527 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
@@ -349,6 +349,14 @@ public class TsFileIOWriter implements AutoCloseable {
logger.info("Time for flushing metadata is {} ms", cost);
}
+ protected void updateCache(String device, TimeseriesMetadata timeseriesMetadata) {
+ // do nothing
+ }
+
+ protected void updateCache(BloomFilter bloomFilter) {
+ // do nothing
+ }
+
private void checkInMemoryPathCount() {
for (ChunkGroupMetadata chunkGroupMetadata : chunkGroupMetadataList) {
pathCount += chunkGroupMetadata.getChunkMetadataList().size();
@@ -426,6 +434,8 @@ public class TsFileIOWriter implements AutoCloseable {
seriesIdxForCurrDevice++;
// serialize the timeseries metadata to file
timeseriesMetadata.serializeTo(out.wrapAsStream());
+ timeseriesMetadata.setChunkMetadataListBuffer(null);
+ updateCache(currentDevice, timeseriesMetadata);
}
addCurrentIndexNodeToQueue(currentIndexNode, measurementMetadataIndexQueue, out);
@@ -444,6 +454,7 @@ public class TsFileIOWriter implements AutoCloseable {
int size = tsFileMetadata.serializeTo(out.wrapAsStream());
size += tsFileMetadata.serializeBloomFilter(out.wrapAsStream(), filter);
+ updateCache(filter);
// write TsFileMetaData size
ReadWriteIOUtils.write(size, out.wrapAsStream());
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/tsmiterator/TSMIterator.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/tsmiterator/TSMIterator.java
index 5d17e0ab7e9..d4c2f1ebcd7 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/tsmiterator/TSMIterator.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/tsmiterator/TSMIterator.java
@@ -95,16 +95,14 @@ public class TSMIterator {
seriesStatistics.mergeStatistics(chunkMetadata.getStatistics());
}
- TimeseriesMetadata timeseriesMetadata =
- new TimeseriesMetadata(
- (byte)
- ((serializeStatistic ? (byte) 1 : (byte) 0) | chunkMetadataList.get(0).getMask()),
- chunkMetadataListLength,
- measurementId,
- dataType,
- seriesStatistics,
- publicBAOS);
- return timeseriesMetadata;
+ return new TimeseriesMetadata(
+ (byte) ((serializeStatistic ? (byte) 1 : (byte) 0) | chunkMetadataList.get(0).getMask()),
+ chunkMetadataListLength,
+ measurementId,
+ dataType,
+ seriesStatistics,
+ publicBAOS,
+ (ArrayList<IChunkMetadata>) chunkMetadataList);
}
public static List<Pair<Path, List<IChunkMetadata>>> sortChunkMetadata(